{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applications clocks set to \"(MEM 877, SM 1530)\" for GPU 00000000:00:1E.0\n",
      "All done.\n"
     ]
    }
   ],
   "source": [
    "!sudo nvidia-persistenced\n",
    "!sudo nvidia-smi -ac 877,1530"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container {width:95% !important;}</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container {width:95% !important;}</style>\"))\n",
    "\n",
    "from core import *\n",
    "from torch_backend import *\n",
    "\n",
    "colors = ColorMap()\n",
    "draw = lambda graph: display(DotGraph({p: ({'fillcolor': colors[type(v)], 'tooltip': repr(v)}, inputs) for p, (v, inputs) in graph.items() if v is not None}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Network definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_norm = partial(BatchNorm, weight_init=None, bias_init=None)\n",
    "\n",
    "def res_block(c_in, c_out, stride, **kw):\n",
    "    block = {\n",
    "        'bn1': batch_norm(c_in, **kw),\n",
    "        'relu1': nn.ReLU(True),\n",
    "        'branch': {\n",
    "            'conv1': nn.Conv2d(c_in, c_out, kernel_size=3, stride=stride, padding=1, bias=False),\n",
    "            'bn2': batch_norm(c_out, **kw),\n",
    "            'relu2': nn.ReLU(True),\n",
    "            'conv2': nn.Conv2d(c_out, c_out, kernel_size=3, stride=1, padding=1, bias=False),\n",
    "        }\n",
    "    }\n",
    "    projection = (stride != 1) or (c_in != c_out)    \n",
    "    if projection:\n",
    "        block['conv3'] = (nn.Conv2d(c_in, c_out, kernel_size=1, stride=stride, padding=0, bias=False), ['relu1'])\n",
    "    block['add'] =  (Add(), [('conv3' if projection else 'relu1'), 'branch/conv2'])\n",
    "    return block\n",
    "\n",
    "def DAWN_net(c=64, block=res_block, prep_bn_relu=False, concat_pool=True, **kw):    \n",
    "    if isinstance(c, int):\n",
    "        c = [c, 2*c, 4*c, 4*c]\n",
    "        \n",
    "    classifier_pool = {\n",
    "        'in': Identity(),\n",
    "        'maxpool': nn.MaxPool2d(4),\n",
    "        'avgpool': (nn.AvgPool2d(4), ['in']),\n",
    "        'concat': (Concat(), ['maxpool', 'avgpool']),\n",
    "    } if concat_pool else {'pool': nn.MaxPool2d(4)}\n",
    "    \n",
    "    return {\n",
    "        'input': (None, []),\n",
    "        'prep': union({'conv': nn.Conv2d(3, c[0], kernel_size=3, stride=1, padding=1, bias=False)},\n",
    "                      {'bn': batch_norm(c[0], **kw), 'relu': nn.ReLU(True)} if prep_bn_relu else {}),\n",
    "        'layer1': {\n",
    "            'block0': block(c[0], c[0], 1, **kw),\n",
    "            'block1': block(c[0], c[0], 1, **kw),\n",
    "        },\n",
    "        'layer2': {\n",
    "            'block0': block(c[0], c[1], 2, **kw),\n",
    "            'block1': block(c[1], c[1], 1, **kw),\n",
    "        },\n",
    "        'layer3': {\n",
    "            'block0': block(c[1], c[2], 2, **kw),\n",
    "            'block1': block(c[2], c[2], 1, **kw),\n",
    "        },\n",
    "        'layer4': {\n",
    "            'block0': block(c[2], c[3], 2, **kw),\n",
    "            'block1': block(c[3], c[3], 1, **kw),\n",
    "        },\n",
    "        'final': union(classifier_pool, {\n",
    "            'flatten': Flatten(),\n",
    "            'linear': nn.Linear(2*c[3] if concat_pool else c[3], 10, bias=True),\n",
    "        }),\n",
    "        'logits': Identity(),\n",
    "    }\n",
    "\n",
    "\n",
    "def conv_bn(c_in, c_out, bn_weight_init=1.0, **kw):\n",
    "    return {\n",
    "        'conv': nn.Conv2d(c_in, c_out, kernel_size=3, stride=1, padding=1, bias=False), \n",
    "        'bn': batch_norm(c_out, bn_weight_init=bn_weight_init, **kw), \n",
    "        'relu': nn.ReLU(True)\n",
    "    }\n",
    "\n",
    "def basic_net(channels, weight,  pool, **kw):\n",
    "    return {\n",
    "        'input': (None, []),\n",
    "        'prep': conv_bn(3, channels['prep'], **kw),\n",
    "        'layer1': dict(conv_bn(channels['prep'], channels['layer1'], **kw), pool=pool),\n",
    "        'layer2': dict(conv_bn(channels['layer1'], channels['layer2'], **kw), pool=pool),\n",
    "        'layer3': dict(conv_bn(channels['layer2'], channels['layer3'], **kw), pool=pool),\n",
    "        'pool': nn.MaxPool2d(4),\n",
    "        'flatten': Flatten(),\n",
    "        'linear': nn.Linear(channels['layer3'], 10, bias=False),\n",
    "        'logits': Mul(weight),\n",
    "    }\n",
    "\n",
    "def net(channels=None, weight=0.125, pool=nn.MaxPool2d(2), extra_layers=(), res_layers=('layer1', 'layer3'), **kw):\n",
    "    channels = channels or {'prep': 64, 'layer1': 128, 'layer2': 256, 'layer3': 512}\n",
    "    residual = lambda c, **kw: {'in': Identity(), 'res1': conv_bn(c, c, **kw), 'res2': conv_bn(c, c, **kw), \n",
    "                                'add': (Add(), ['in', 'res2/relu'])}\n",
    "    n = basic_net(channels, weight, pool, **kw)\n",
    "    for layer in res_layers:\n",
    "        n[layer]['residual'] = residual(channels[layer], **kw)\n",
    "    for layer in extra_layers:\n",
    "        n[layer]['extra'] = conv_bn(channels[layer], channels[layer], **kw)       \n",
    "    return n\n",
    "\n",
    "remove_identity_nodes = lambda net: remove_by_type(net, Identity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download and preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Preprocessing training data\n",
      "Finished in 3.1 seconds\n",
      "Preprocessing test data\n",
      "Finished in 0.14 seconds\n"
     ]
    }
   ],
   "source": [
    "DATA_DIR = './data'\n",
    "dataset = cifar10(DATA_DIR)\n",
    "timer = Timer()\n",
    "print('Preprocessing training data')\n",
    "transforms = [\n",
    "    partial(normalise, mean=np.array(cifar10_mean, dtype=np.float32), std=np.array(cifar10_std, dtype=np.float32)),\n",
    "    partial(transpose, source='NHWC', target='NCHW'), \n",
    "]\n",
    "train_set = list(zip(*preprocess(dataset['train'], [partial(pad, border=4)] + transforms).values()))\n",
    "print(f'Finished in {timer():.2} seconds')\n",
    "print('Preprocessing test data')\n",
    "test_set = list(zip(*preprocess(dataset['valid'], transforms).values()))\n",
    "print(f'Finished in {timer():.2} seconds')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, lr_schedule, train_set, test_set, batch_size, num_workers=0):\n",
    "    train_batches = DataLoader(train_set, batch_size, shuffle=True, set_random_choices=True, num_workers=num_workers)\n",
    "    test_batches = DataLoader(test_set, batch_size, shuffle=False, num_workers=num_workers)\n",
    "    \n",
    "    lr = lambda step: lr_schedule(step/len(train_batches))/batch_size\n",
    "    opts = [SGD(trainable_params(model).values(), {'lr': lr, 'weight_decay': Const(5e-4*batch_size), 'momentum': Const(0.9)})]\n",
    "    logs, state = Table(), {MODEL: model, LOSS: x_ent_loss, OPTS: opts}\n",
    "    for epoch in range(lr_schedule.knots[-1]):\n",
    "        logs.append(union({'epoch': epoch+1, 'lr': lr_schedule(epoch+1)}, \n",
    "                          train_epoch(state, Timer(torch.cuda.synchronize), train_batches, test_batches)))\n",
    "    return logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 1: Baseline](https://www.myrtle.ai/2018/09/24/how_to_train_your_resnet_1/) - DAWNbench baseline + no initial bn-relu+ efficient dataloading/augmentation, 1 dataloader process (301s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"43pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 43.08\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.187305 0.187305) rotate(0) translate(4 226)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-226 5762,-226 5762,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-52C94,-52 140,-52 140,-52 146,-52 152,-58 152,-64 152,-64 152,-115 152,-115 152,-121 146,-127 140,-127 140,-127 94,-127 94,-127 88,-127 82,-121 82,-115 82,-115 82,-64 82,-64 82,-58 88,-52 94,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M176,-16C176,-16 1408,-16 1408,-16 1414,-16 1420,-22 1420,-28 1420,-28 1420,-167 1420,-167 1420,-173 1414,-179 1408,-179 1408,-179 176,-179 176,-179 170,-179 164,-173 164,-167 164,-167 164,-28 164,-28 164,-22 170,-16 176,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"792\" y=\"-163.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer1_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M184,-24C184,-24 770,-24 770,-24 776,-24 782,-30 782,-36 782,-36 782,-136 782,-136 782,-142 776,-148 770,-148 770,-148 184,-148 184,-148 178,-148 172,-142 172,-136 172,-136 172,-36 172,-36 172,-30 178,-24 184,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer1_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-32C364,-32 680,-32 680,-32 686,-32 692,-38 692,-44 692,-44 692,-95 692,-95 692,-101 686,-107 680,-107 680,-107 364,-107 364,-107 358,-107 352,-101 352,-95 352,-95 352,-44 352,-44 352,-38 358,-32 364,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer1_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M814,-24C814,-24 1400,-24 1400,-24 1406,-24 1412,-30 1412,-36 1412,-36 1412,-136 1412,-136 1412,-142 1406,-148 1400,-148 1400,-148 814,-148 814,-148 808,-148 802,-142 802,-136 802,-136 802,-36 802,-36 802,-30 808,-24 814,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer1_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M994,-32C994,-32 1310,-32 1310,-32 1316,-32 1322,-38 1322,-44 1322,-44 1322,-95 1322,-95 1322,-101 1316,-107 1310,-107 1310,-107 994,-107 994,-107 988,-107 982,-101 982,-95 982,-95 982,-44 982,-44 982,-38 988,-32 994,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"1152\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1440,-17C1440,-17 2672,-17 2672,-17 2678,-17 2684,-23 2684,-29 2684,-29 2684,-202 2684,-202 2684,-208 2678,-214 2672,-214 2672,-214 1440,-214 1440,-214 1434,-214 1428,-208 1428,-202 1428,-202 1428,-29 1428,-29 1428,-23 1434,-17 1440,-17\"/>\n",
       "<text text-anchor=\"middle\" x=\"2056\" y=\"-198.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_layer2_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1448,-25C1448,-25 2034,-25 2034,-25 2040,-25 2046,-31 2046,-37 2046,-37 2046,-171 2046,-171 2046,-177 2040,-183 2034,-183 2034,-183 1448,-183 1448,-183 1442,-183 1436,-177 1436,-171 1436,-171 1436,-37 1436,-37 1436,-31 1442,-25 1448,-25\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-167.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_layer2_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1628,-33C1628,-33 1944,-33 1944,-33 1950,-33 1956,-39 1956,-45 1956,-45 1956,-96 1956,-96 1956,-102 1950,-108 1944,-108 1944,-108 1628,-108 1628,-108 1622,-108 1616,-102 1616,-96 1616,-96 1616,-45 1616,-45 1616,-39 1622,-33 1628,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"1786\" y=\"-92.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_layer2_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2078,-30C2078,-30 2664,-30 2664,-30 2670,-30 2676,-36 2676,-42 2676,-42 2676,-142 2676,-142 2676,-148 2670,-154 2664,-154 2664,-154 2078,-154 2078,-154 2072,-154 2066,-148 2066,-142 2066,-142 2066,-42 2066,-42 2066,-36 2072,-30 2078,-30\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-138.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust11\" class=\"cluster\"><title>cluster_layer2_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2258,-38C2258,-38 2574,-38 2574,-38 2580,-38 2586,-44 2586,-50 2586,-50 2586,-101 2586,-101 2586,-107 2580,-113 2574,-113 2574,-113 2258,-113 2258,-113 2252,-113 2246,-107 2246,-101 2246,-101 2246,-50 2246,-50 2246,-44 2252,-38 2258,-38\"/>\n",
       "<text text-anchor=\"middle\" x=\"2416\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust12\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2704,-12C2704,-12 3936,-12 3936,-12 3942,-12 3948,-18 3948,-24 3948,-24 3948,-197 3948,-197 3948,-203 3942,-209 3936,-209 3936,-209 2704,-209 2704,-209 2698,-209 2692,-203 2692,-197 2692,-197 2692,-24 2692,-24 2692,-18 2698,-12 2704,-12\"/>\n",
       "<text text-anchor=\"middle\" x=\"3320\" y=\"-193.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust13\" class=\"cluster\"><title>cluster_layer3_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2712,-20C2712,-20 3298,-20 3298,-20 3304,-20 3310,-26 3310,-32 3310,-32 3310,-166 3310,-166 3310,-172 3304,-178 3298,-178 3298,-178 2712,-178 2712,-178 2706,-178 2700,-172 2700,-166 2700,-166 2700,-32 2700,-32 2700,-26 2706,-20 2712,-20\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-162.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust14\" class=\"cluster\"><title>cluster_layer3_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2892,-28C2892,-28 3208,-28 3208,-28 3214,-28 3220,-34 3220,-40 3220,-40 3220,-91 3220,-91 3220,-97 3214,-103 3208,-103 3208,-103 2892,-103 2892,-103 2886,-103 2880,-97 2880,-91 2880,-91 2880,-40 2880,-40 2880,-34 2886,-28 2892,-28\"/>\n",
       "<text text-anchor=\"middle\" x=\"3050\" y=\"-87.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust15\" class=\"cluster\"><title>cluster_layer3_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3342,-37C3342,-37 3928,-37 3928,-37 3934,-37 3940,-43 3940,-49 3940,-49 3940,-149 3940,-149 3940,-155 3934,-161 3928,-161 3928,-161 3342,-161 3342,-161 3336,-161 3330,-155 3330,-149 3330,-149 3330,-49 3330,-49 3330,-43 3336,-37 3342,-37\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-145.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust16\" class=\"cluster\"><title>cluster_layer3_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3522,-45C3522,-45 3838,-45 3838,-45 3844,-45 3850,-51 3850,-57 3850,-57 3850,-108 3850,-108 3850,-114 3844,-120 3838,-120 3838,-120 3522,-120 3522,-120 3516,-120 3510,-114 3510,-108 3510,-108 3510,-57 3510,-57 3510,-51 3516,-45 3522,-45\"/>\n",
       "<text text-anchor=\"middle\" x=\"3680\" y=\"-104.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust17\" class=\"cluster\"><title>cluster_layer4</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3968,-8C3968,-8 5200,-8 5200,-8 5206,-8 5212,-14 5212,-20 5212,-20 5212,-193 5212,-193 5212,-199 5206,-205 5200,-205 5200,-205 3968,-205 3968,-205 3962,-205 3956,-199 3956,-193 3956,-193 3956,-20 3956,-20 3956,-14 3962,-8 3968,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"4584\" y=\"-189.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer4</text>\n",
       "</g>\n",
       "<g id=\"clust18\" class=\"cluster\"><title>cluster_layer4_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3976,-16C3976,-16 4562,-16 4562,-16 4568,-16 4574,-22 4574,-28 4574,-28 4574,-162 4574,-162 4574,-168 4568,-174 4562,-174 4562,-174 3976,-174 3976,-174 3970,-174 3964,-168 3964,-162 3964,-162 3964,-28 3964,-28 3964,-22 3970,-16 3976,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-158.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust19\" class=\"cluster\"><title>cluster_layer4_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4156,-24C4156,-24 4472,-24 4472,-24 4478,-24 4484,-30 4484,-36 4484,-36 4484,-87 4484,-87 4484,-93 4478,-99 4472,-99 4472,-99 4156,-99 4156,-99 4150,-99 4144,-93 4144,-87 4144,-87 4144,-36 4144,-36 4144,-30 4150,-24 4156,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"4314\" y=\"-83.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust20\" class=\"cluster\"><title>cluster_layer4_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4606,-33C4606,-33 5192,-33 5192,-33 5198,-33 5204,-39 5204,-45 5204,-45 5204,-145 5204,-145 5204,-151 5198,-157 5192,-157 5192,-157 4606,-157 4606,-157 4600,-157 4594,-151 4594,-145 4594,-145 4594,-45 4594,-45 4594,-39 4600,-33 4606,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-141.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust21\" class=\"cluster\"><title>cluster_layer4_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4786,-41C4786,-41 5102,-41 5102,-41 5108,-41 5114,-47 5114,-53 5114,-53 5114,-104 5114,-104 5114,-110 5108,-116 5102,-116 5102,-116 4786,-116 4786,-116 4780,-116 4774,-110 4774,-104 4774,-104 4774,-53 4774,-53 4774,-47 4780,-41 4786,-41\"/>\n",
       "<text text-anchor=\"middle\" x=\"4944\" y=\"-100.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust22\" class=\"cluster\"><title>cluster_final</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M5236,-34C5236,-34 5653,-34 5653,-34 5659,-34 5665,-40 5665,-46 5665,-46 5665,-151 5665,-151 5665,-157 5659,-163 5653,-163 5653,-163 5236,-163 5236,-163 5230,-163 5224,-157 5224,-151 5224,-151 5224,-46 5224,-46 5224,-40 5230,-34 5236,-34\"/>\n",
       "<text text-anchor=\"middle\" x=\"5444.5\" y=\"-147.8\" font-family=\"Times,serif\" font-size=\"14.00\">final</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-96C132,-96 102,-96 102,-96 96,-96 90,-90 90,-84 90,-84 90,-72 90,-72 90,-66 96,-60 102,-60 102,-60 132,-60 132,-60 138,-60 144,-66 144,-72 144,-72 144,-84 144,-84 144,-90 138,-96 132,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1 -->\n",
       "<g id=\"node2\" class=\"node\"><title>layer1_block0_bn1</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-96C222,-96 192,-96 192,-96 186,-96 180,-90 180,-84 180,-84 180,-72 180,-72 180,-66 186,-60 192,-60 192,-60 222,-60 222,-60 228,-60 234,-66 234,-72 234,-72 234,-84 234,-84 234,-90 228,-96 222,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;layer1_block0_bn1 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;layer1_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-78C152.393,-78 161.311,-78 169.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-81.5001 179.919,-78 169.919,-74.5001 169.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>layer1_block0_relu1</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-96C312,-96 282,-96 282,-96 276,-96 270,-90 270,-84 270,-84 270,-72 270,-72 270,-66 276,-60 282,-60 282,-60 312,-60 312,-60 318,-60 324,-66 324,-72 324,-72 324,-84 324,-84 324,-90 318,-96 312,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1&#45;&gt;layer1_block0_relu1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>layer1_block0_bn1&#45;&gt;layer1_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-78C242.393,-78 251.311,-78 259.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-81.5001 269.919,-78 259.919,-74.5001 259.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_block0_branch_conv1</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-76C402,-76 372,-76 372,-76 366,-76 360,-70 360,-64 360,-64 360,-52 360,-52 360,-46 366,-40 372,-40 372,-40 402,-40 402,-40 408,-40 414,-46 414,-52 414,-52 414,-64 414,-64 414,-70 408,-76 402,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-71.9993C332.481,-70.1634 341.507,-68.1121 350.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"350.943,-69.5568 359.919,-63.9275 349.392,-62.7308 350.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_add -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer1_block0_add</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M762,-96C762,-96 732,-96 732,-96 726,-96 720,-90 720,-84 720,-84 720,-72 720,-72 720,-66 726,-60 732,-60 732,-60 762,-60 762,-60 768,-60 774,-66 774,-72 774,-72 774,-84 774,-84 774,-90 768,-96 762,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"747\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M321.737,-96.1018C330.735,-101.964 341.381,-107.78 352,-111 404.782,-127.006 420.844,-116 476,-116 476,-116 476,-116 568,-116 623.156,-116 639.218,-127.006 692,-111 699.301,-108.786 706.614,-105.345 713.396,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"715.545,-104.295 722.263,-96.1018 711.902,-98.3176 715.545,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2 -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_block0_branch_bn2</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-76C492,-76 462,-76 462,-76 456,-76 450,-70 450,-64 450,-64 450,-52 450,-52 450,-46 456,-40 462,-40 462,-40 492,-40 492,-40 498,-40 504,-46 504,-52 504,-52 504,-64 504,-64 504,-70 498,-76 492,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-58C422.393,-58 431.311,-58 439.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-61.5001 449.919,-58 439.919,-54.5001 439.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2 -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer1_block0_branch_relu2</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-76C582,-76 552,-76 552,-76 546,-76 540,-70 540,-64 540,-64 540,-52 540,-52 540,-46 546,-40 552,-40 552,-40 582,-40 582,-40 588,-40 594,-46 594,-52 594,-52 594,-64 594,-64 594,-70 588,-76 582,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-58C512.393,-58 521.311,-58 529.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-61.5001 539.919,-58 529.919,-54.5001 529.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2 -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer1_block0_branch_conv2</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M672,-76C672,-76 642,-76 642,-76 636,-76 630,-70 630,-64 630,-64 630,-52 630,-52 630,-46 636,-40 642,-40 642,-40 672,-40 672,-40 678,-40 684,-46 684,-52 684,-52 684,-64 684,-64 684,-70 678,-76 672,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-58C602.393,-58 611.311,-58 619.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-61.5001 629.919,-58 619.919,-54.5001 619.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer1_block0_branch_conv2&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.403,-64.0007C692.481,-65.8366 701.507,-67.8879 710.105,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"709.392,-73.2692 719.919,-72.0725 710.943,-66.4432 709.392,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1 -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer1_block1_bn1</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M852,-96C852,-96 822,-96 822,-96 816,-96 810,-90 810,-84 810,-84 810,-72 810,-72 810,-66 816,-60 822,-60 822,-60 852,-60 852,-60 858,-60 864,-66 864,-72 864,-72 864,-84 864,-84 864,-90 858,-96 852,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_add&#45;&gt;layer1_block1_bn1 -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer1_block0_add&#45;&gt;layer1_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.403,-78C782.393,-78 791.311,-78 799.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"799.919,-81.5001 809.919,-78 799.919,-74.5001 799.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1 -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer1_block1_relu1</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M942,-96C942,-96 912,-96 912,-96 906,-96 900,-90 900,-84 900,-84 900,-72 900,-72 900,-66 906,-60 912,-60 912,-60 942,-60 942,-60 948,-60 954,-66 954,-72 954,-72 954,-84 954,-84 954,-90 948,-96 942,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1&#45;&gt;layer1_block1_relu1 -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer1_block1_bn1&#45;&gt;layer1_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.403,-78C872.393,-78 881.311,-78 889.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"889.919,-81.5001 899.919,-78 889.919,-74.5001 889.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1 -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer1_block1_branch_conv1</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1032,-76C1032,-76 1002,-76 1002,-76 996,-76 990,-70 990,-64 990,-64 990,-52 990,-52 990,-46 996,-40 1002,-40 1002,-40 1032,-40 1032,-40 1038,-40 1044,-46 1044,-52 1044,-52 1044,-64 1044,-64 1044,-70 1038,-76 1032,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1 -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M954.403,-71.9993C962.481,-70.1634 971.507,-68.1121 980.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"980.943,-69.5568 989.919,-63.9275 979.392,-62.7308 980.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_add -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer1_block1_add</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1392,-96C1392,-96 1362,-96 1362,-96 1356,-96 1350,-90 1350,-84 1350,-84 1350,-72 1350,-72 1350,-66 1356,-60 1362,-60 1362,-60 1392,-60 1392,-60 1398,-60 1404,-66 1404,-72 1404,-72 1404,-84 1404,-84 1404,-90 1398,-96 1392,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1377\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M951.737,-96.1018C960.735,-101.964 971.381,-107.78 982,-111 1034.78,-127.006 1050.84,-116 1106,-116 1106,-116 1106,-116 1198,-116 1253.16,-116 1269.22,-127.006 1322,-111 1329.3,-108.786 1336.61,-105.345 1343.4,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1345.55,-104.295 1352.26,-96.1018 1341.9,-98.3176 1345.55,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2 -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer1_block1_branch_bn2</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1122,-76C1122,-76 1092,-76 1092,-76 1086,-76 1080,-70 1080,-64 1080,-64 1080,-52 1080,-52 1080,-46 1086,-40 1092,-40 1092,-40 1122,-40 1122,-40 1128,-40 1134,-46 1134,-52 1134,-52 1134,-64 1134,-64 1134,-70 1128,-76 1122,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2 -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.4,-58C1052.39,-58 1061.31,-58 1069.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1069.92,-61.5001 1079.92,-58 1069.92,-54.5001 1069.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2 -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer1_block1_branch_relu2</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1212,-76C1212,-76 1182,-76 1182,-76 1176,-76 1170,-70 1170,-64 1170,-64 1170,-52 1170,-52 1170,-46 1176,-40 1182,-40 1182,-40 1212,-40 1212,-40 1218,-40 1224,-46 1224,-52 1224,-52 1224,-64 1224,-64 1224,-70 1218,-76 1212,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2 -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1134.4,-58C1142.39,-58 1151.31,-58 1159.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1159.92,-61.5001 1169.92,-58 1159.92,-54.5001 1159.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2 -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer1_block1_branch_conv2</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1302,-76C1302,-76 1272,-76 1272,-76 1266,-76 1260,-70 1260,-64 1260,-64 1260,-52 1260,-52 1260,-46 1266,-40 1272,-40 1272,-40 1302,-40 1302,-40 1308,-40 1314,-46 1314,-52 1314,-52 1314,-64 1314,-64 1314,-70 1308,-76 1302,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1287\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2 -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1224.4,-58C1232.39,-58 1241.31,-58 1249.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1249.92,-61.5001 1259.92,-58 1249.92,-54.5001 1249.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>layer1_block1_branch_conv2&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1314.4,-64.0007C1322.48,-65.8366 1331.51,-67.8879 1340.1,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1339.39,-73.2692 1349.92,-72.0725 1340.94,-66.4432 1339.39,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1 -->\n",
       "<g id=\"node16\" class=\"node\"><title>layer2_block0_bn1</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1486,-96C1486,-96 1456,-96 1456,-96 1450,-96 1444,-90 1444,-84 1444,-84 1444,-72 1444,-72 1444,-66 1450,-60 1456,-60 1456,-60 1486,-60 1486,-60 1492,-60 1498,-66 1498,-72 1498,-72 1498,-84 1498,-84 1498,-90 1492,-96 1486,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1471\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_add&#45;&gt;layer2_block0_bn1 -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>layer1_block1_add&#45;&gt;layer2_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1404.12,-78C1413.34,-78 1423.89,-78 1433.8,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1433.82,-81.5001 1443.82,-78 1433.82,-74.5001 1433.82,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1 -->\n",
       "<g id=\"node17\" class=\"node\"><title>layer2_block0_relu1</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1576,-96C1576,-96 1546,-96 1546,-96 1540,-96 1534,-90 1534,-84 1534,-84 1534,-72 1534,-72 1534,-66 1540,-60 1546,-60 1546,-60 1576,-60 1576,-60 1582,-60 1588,-66 1588,-72 1588,-72 1588,-84 1588,-84 1588,-90 1582,-96 1576,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1561\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1&#45;&gt;layer2_block0_relu1 -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>layer2_block0_bn1&#45;&gt;layer2_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1498.4,-78C1506.39,-78 1515.31,-78 1523.82,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1523.92,-81.5001 1533.92,-78 1523.92,-74.5001 1523.92,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1 -->\n",
       "<g id=\"node18\" class=\"node\"><title>layer2_block0_branch_conv1</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1666,-77C1666,-77 1636,-77 1636,-77 1630,-77 1624,-71 1624,-65 1624,-65 1624,-53 1624,-53 1624,-47 1630,-41 1636,-41 1636,-41 1666,-41 1666,-41 1672,-41 1678,-47 1678,-53 1678,-53 1678,-65 1678,-65 1678,-71 1672,-77 1666,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1651\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1 -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1588.4,-72.2994C1596.48,-70.5552 1605.51,-68.6065 1614.1,-66.7501\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1614.88,-70.1628 1623.92,-64.6311 1613.41,-63.3205 1614.88,-70.1628\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3 -->\n",
       "<g id=\"node22\" class=\"node\"><title>layer2_block0_conv3</title>\n",
       "<g id=\"a_node22\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-152C1936,-152 1906,-152 1906,-152 1900,-152 1894,-146 1894,-140 1894,-140 1894,-128 1894,-128 1894,-122 1900,-116 1906,-116 1906,-116 1936,-116 1936,-116 1942,-116 1948,-122 1948,-128 1948,-128 1948,-140 1948,-140 1948,-146 1942,-152 1936,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-130.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_conv3 -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1585.19,-96.1046C1594.3,-102.243 1605.16,-108.442 1616,-112 1708.77,-142.443 1825.74,-140.431 1883.83,-136.933\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1884.22,-140.415 1893.97,-136.27 1883.76,-133.43 1884.22,-140.415\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2 -->\n",
       "<g id=\"node19\" class=\"node\"><title>layer2_block0_branch_bn2</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1756,-77C1756,-77 1726,-77 1726,-77 1720,-77 1714,-71 1714,-65 1714,-65 1714,-53 1714,-53 1714,-47 1720,-41 1726,-41 1726,-41 1756,-41 1756,-41 1762,-41 1768,-47 1768,-53 1768,-53 1768,-65 1768,-65 1768,-71 1762,-77 1756,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2 -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1678.4,-59C1686.39,-59 1695.31,-59 1703.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1703.92,-62.5001 1713.92,-59 1703.92,-55.5001 1703.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2 -->\n",
       "<g id=\"node20\" class=\"node\"><title>layer2_block0_branch_relu2</title>\n",
       "<g id=\"a_node20\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1846,-77C1846,-77 1816,-77 1816,-77 1810,-77 1804,-71 1804,-65 1804,-65 1804,-53 1804,-53 1804,-47 1810,-41 1816,-41 1816,-41 1846,-41 1846,-41 1852,-41 1858,-47 1858,-53 1858,-53 1858,-65 1858,-65 1858,-71 1852,-77 1846,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1831\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2 -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1768.4,-59C1776.39,-59 1785.31,-59 1793.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1793.92,-62.5001 1803.92,-59 1793.92,-55.5001 1793.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2 -->\n",
       "<g id=\"node21\" class=\"node\"><title>layer2_block0_branch_conv2</title>\n",
       "<g id=\"a_node21\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-77C1936,-77 1906,-77 1906,-77 1900,-77 1894,-71 1894,-65 1894,-65 1894,-53 1894,-53 1894,-47 1900,-41 1906,-41 1906,-41 1936,-41 1936,-41 1942,-41 1948,-47 1948,-53 1948,-53 1948,-65 1948,-65 1948,-71 1942,-77 1936,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2 -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1858.4,-59C1866.39,-59 1875.31,-59 1883.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1883.92,-62.5001 1893.92,-59 1883.92,-55.5001 1883.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_add -->\n",
       "<g id=\"node23\" class=\"node\"><title>layer2_block0_add</title>\n",
       "<g id=\"a_node23\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2026,-97C2026,-97 1996,-97 1996,-97 1990,-97 1984,-91 1984,-85 1984,-85 1984,-73 1984,-73 1984,-67 1990,-61 1996,-61 1996,-61 2026,-61 2026,-61 2032,-61 2038,-67 2038,-73 2038,-73 2038,-85 2038,-85 2038,-91 2032,-97 2026,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2011\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>layer2_block0_branch_conv2&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-65.0007C1956.48,-66.8366 1965.51,-68.8879 1974.1,-70.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1973.39,-74.2692 1983.92,-73.0725 1974.94,-67.4432 1973.39,-74.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>layer2_block0_conv3&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-117.498C1956.83,-112.23 1966.29,-106.317 1975.22,-100.735\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1977.29,-103.569 1983.92,-95.3006 1973.58,-97.6327 1977.29,-103.569\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1 -->\n",
       "<g id=\"node24\" class=\"node\"><title>layer2_block1_bn1</title>\n",
       "<g id=\"a_node24\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2116,-97C2116,-97 2086,-97 2086,-97 2080,-97 2074,-91 2074,-85 2074,-85 2074,-73 2074,-73 2074,-67 2080,-61 2086,-61 2086,-61 2116,-61 2116,-61 2122,-61 2128,-67 2128,-73 2128,-73 2128,-85 2128,-85 2128,-91 2122,-97 2116,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2101\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_add&#45;&gt;layer2_block1_bn1 -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>layer2_block0_add&#45;&gt;layer2_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2038.4,-79C2046.39,-79 2055.31,-79 2063.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2063.92,-82.5001 2073.92,-79 2063.92,-75.5001 2063.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1 -->\n",
       "<g id=\"node25\" class=\"node\"><title>layer2_block1_relu1</title>\n",
       "<g id=\"a_node25\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2206,-97C2206,-97 2176,-97 2176,-97 2170,-97 2164,-91 2164,-85 2164,-85 2164,-73 2164,-73 2164,-67 2170,-61 2176,-61 2176,-61 2206,-61 2206,-61 2212,-61 2218,-67 2218,-73 2218,-73 2218,-85 2218,-85 2218,-91 2212,-97 2206,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2191\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1&#45;&gt;layer2_block1_relu1 -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>layer2_block1_bn1&#45;&gt;layer2_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2128.4,-79C2136.39,-79 2145.31,-79 2153.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2153.92,-82.5001 2163.92,-79 2153.92,-75.5001 2153.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1 -->\n",
       "<g id=\"node26\" class=\"node\"><title>layer2_block1_branch_conv1</title>\n",
       "<g id=\"a_node26\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2296,-82C2296,-82 2266,-82 2266,-82 2260,-82 2254,-76 2254,-70 2254,-70 2254,-58 2254,-58 2254,-52 2260,-46 2266,-46 2266,-46 2296,-46 2296,-46 2302,-46 2308,-52 2308,-58 2308,-58 2308,-70 2308,-70 2308,-76 2302,-82 2296,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2281\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1 -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2218.4,-74.4995C2226.39,-73.1375 2235.31,-71.6175 2243.82,-70.1664\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2244.65,-73.5763 2253.92,-68.4456 2243.47,-66.6758 2244.65,-73.5763\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_add -->\n",
       "<g id=\"node30\" class=\"node\"><title>layer2_block1_add</title>\n",
       "<g id=\"a_node30\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2656,-82C2656,-82 2626,-82 2626,-82 2620,-82 2614,-76 2614,-70 2614,-70 2614,-58 2614,-58 2614,-52 2620,-46 2626,-46 2626,-46 2656,-46 2656,-46 2662,-46 2668,-52 2668,-58 2668,-58 2668,-70 2668,-70 2668,-76 2662,-82 2656,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2641\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge33\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2211.74,-97.0337C2221.35,-104.671 2233.51,-112.769 2246,-117 2298.24,-134.703 2314.84,-122 2370,-122 2370,-122 2370,-122 2462,-122 2517.16,-122 2535.38,-138.893 2586,-117 2599.55,-111.141 2611.6,-100.216 2620.87,-89.7996\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2623.65,-91.9369 2627.39,-82.0259 2618.28,-87.4385 2623.65,-91.9369\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2 -->\n",
       "<g id=\"node27\" class=\"node\"><title>layer2_block1_branch_bn2</title>\n",
       "<g id=\"a_node27\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2386,-82C2386,-82 2356,-82 2356,-82 2350,-82 2344,-76 2344,-70 2344,-70 2344,-58 2344,-58 2344,-52 2350,-46 2356,-46 2356,-46 2386,-46 2386,-46 2392,-46 2398,-52 2398,-58 2398,-58 2398,-70 2398,-70 2398,-76 2392,-82 2386,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2 -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2308.4,-64C2316.39,-64 2325.31,-64 2333.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2333.92,-67.5001 2343.92,-64 2333.92,-60.5001 2333.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2 -->\n",
       "<g id=\"node28\" class=\"node\"><title>layer2_block1_branch_relu2</title>\n",
       "<g id=\"a_node28\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2476,-82C2476,-82 2446,-82 2446,-82 2440,-82 2434,-76 2434,-70 2434,-70 2434,-58 2434,-58 2434,-52 2440,-46 2446,-46 2446,-46 2476,-46 2476,-46 2482,-46 2488,-52 2488,-58 2488,-58 2488,-70 2488,-70 2488,-76 2482,-82 2476,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2461\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2 -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2398.4,-64C2406.39,-64 2415.31,-64 2423.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2423.92,-67.5001 2433.92,-64 2423.92,-60.5001 2423.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2 -->\n",
       "<g id=\"node29\" class=\"node\"><title>layer2_block1_branch_conv2</title>\n",
       "<g id=\"a_node29\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2566,-82C2566,-82 2536,-82 2536,-82 2530,-82 2524,-76 2524,-70 2524,-70 2524,-58 2524,-58 2524,-52 2530,-46 2536,-46 2536,-46 2566,-46 2566,-46 2572,-46 2578,-52 2578,-58 2578,-58 2578,-70 2578,-70 2578,-76 2572,-82 2566,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2551\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2 -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2488.4,-64C2496.39,-64 2505.31,-64 2513.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2513.92,-67.5001 2523.92,-64 2513.92,-60.5001 2513.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge34\" class=\"edge\"><title>layer2_block1_branch_conv2&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2578.4,-64C2586.39,-64 2595.31,-64 2603.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2603.92,-67.5001 2613.92,-64 2603.92,-60.5001 2603.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1 -->\n",
       "<g id=\"node31\" class=\"node\"><title>layer3_block0_bn1</title>\n",
       "<g id=\"a_node31\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2750,-82C2750,-82 2720,-82 2720,-82 2714,-82 2708,-76 2708,-70 2708,-70 2708,-58 2708,-58 2708,-52 2714,-46 2720,-46 2720,-46 2750,-46 2750,-46 2756,-46 2762,-52 2762,-58 2762,-58 2762,-70 2762,-70 2762,-76 2756,-82 2750,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2735\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_add&#45;&gt;layer3_block0_bn1 -->\n",
       "<g id=\"edge35\" class=\"edge\"><title>layer2_block1_add&#45;&gt;layer3_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2668.12,-64C2677.34,-64 2687.89,-64 2697.8,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2697.82,-67.5001 2707.82,-64 2697.82,-60.5001 2697.82,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1 -->\n",
       "<g id=\"node32\" class=\"node\"><title>layer3_block0_relu1</title>\n",
       "<g id=\"a_node32\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2840,-82C2840,-82 2810,-82 2810,-82 2804,-82 2798,-76 2798,-70 2798,-70 2798,-58 2798,-58 2798,-52 2804,-46 2810,-46 2810,-46 2840,-46 2840,-46 2846,-46 2852,-52 2852,-58 2852,-58 2852,-70 2852,-70 2852,-76 2846,-82 2840,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2825\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1&#45;&gt;layer3_block0_relu1 -->\n",
       "<g id=\"edge36\" class=\"edge\"><title>layer3_block0_bn1&#45;&gt;layer3_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2762.4,-64C2770.39,-64 2779.31,-64 2787.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2787.92,-67.5001 2797.92,-64 2787.92,-60.5001 2787.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1 -->\n",
       "<g id=\"node33\" class=\"node\"><title>layer3_block0_branch_conv1</title>\n",
       "<g id=\"a_node33\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2930,-72C2930,-72 2900,-72 2900,-72 2894,-72 2888,-66 2888,-60 2888,-60 2888,-48 2888,-48 2888,-42 2894,-36 2900,-36 2900,-36 2930,-36 2930,-36 2936,-36 2942,-42 2942,-48 2942,-48 2942,-60 2942,-60 2942,-66 2936,-72 2930,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"2915\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1 -->\n",
       "<g id=\"edge37\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2852.4,-60.9997C2860.39,-60.0917 2869.31,-59.0783 2877.82,-58.1109\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2878.38,-61.5706 2887.92,-56.9638 2877.59,-54.6153 2878.38,-61.5706\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3 -->\n",
       "<g id=\"node37\" class=\"node\"><title>layer3_block0_conv3</title>\n",
       "<g id=\"a_node37\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-147C3200,-147 3170,-147 3170,-147 3164,-147 3158,-141 3158,-135 3158,-135 3158,-123 3158,-123 3158,-117 3164,-111 3170,-111 3170,-111 3200,-111 3200,-111 3206,-111 3212,-117 3212,-123 3212,-123 3212,-135 3212,-135 3212,-141 3206,-147 3200,-147\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-125.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_conv3 -->\n",
       "<g id=\"edge41\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2843.22,-82.1953C2853.06,-91.336 2866.16,-101.594 2880,-107 2970.95,-142.522 3088.72,-138.267 3147.4,-133.121\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3148.02,-136.578 3157.65,-132.159 3147.37,-129.609 3148.02,-136.578\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2 -->\n",
       "<g id=\"node34\" class=\"node\"><title>layer3_block0_branch_bn2</title>\n",
       "<g id=\"a_node34\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3020,-72C3020,-72 2990,-72 2990,-72 2984,-72 2978,-66 2978,-60 2978,-60 2978,-48 2978,-48 2978,-42 2984,-36 2990,-36 2990,-36 3020,-36 3020,-36 3026,-36 3032,-42 3032,-48 3032,-48 3032,-60 3032,-60 3032,-66 3026,-72 3020,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2 -->\n",
       "<g id=\"edge38\" class=\"edge\"><title>layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2942.4,-54C2950.39,-54 2959.31,-54 2967.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2967.92,-57.5001 2977.92,-54 2967.92,-50.5001 2967.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2 -->\n",
       "<g id=\"node35\" class=\"node\"><title>layer3_block0_branch_relu2</title>\n",
       "<g id=\"a_node35\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3110,-72C3110,-72 3080,-72 3080,-72 3074,-72 3068,-66 3068,-60 3068,-60 3068,-48 3068,-48 3068,-42 3074,-36 3080,-36 3080,-36 3110,-36 3110,-36 3116,-36 3122,-42 3122,-48 3122,-48 3122,-60 3122,-60 3122,-66 3116,-72 3110,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3095\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2 -->\n",
       "<g id=\"edge39\" class=\"edge\"><title>layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3032.4,-54C3040.39,-54 3049.31,-54 3057.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3057.92,-57.5001 3067.92,-54 3057.92,-50.5001 3057.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2 -->\n",
       "<g id=\"node36\" class=\"node\"><title>layer3_block0_branch_conv2</title>\n",
       "<g id=\"a_node36\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-72C3200,-72 3170,-72 3170,-72 3164,-72 3158,-66 3158,-60 3158,-60 3158,-48 3158,-48 3158,-42 3164,-36 3170,-36 3170,-36 3200,-36 3200,-36 3206,-36 3212,-42 3212,-48 3212,-48 3212,-60 3212,-60 3212,-66 3206,-72 3200,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2 -->\n",
       "<g id=\"edge40\" class=\"edge\"><title>layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3122.4,-54C3130.39,-54 3139.31,-54 3147.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3147.92,-57.5001 3157.92,-54 3147.92,-50.5001 3147.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_add -->\n",
       "<g id=\"node38\" class=\"node\"><title>layer3_block0_add</title>\n",
       "<g id=\"a_node38\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3290,-109C3290,-109 3260,-109 3260,-109 3254,-109 3248,-103 3248,-97 3248,-97 3248,-85 3248,-85 3248,-79 3254,-73 3260,-73 3260,-73 3290,-73 3290,-73 3296,-73 3302,-79 3302,-85 3302,-85 3302,-97 3302,-97 3302,-103 3296,-109 3290,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3275\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge43\" class=\"edge\"><title>layer3_block0_branch_conv2&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-65.1012C3220.66,-68.5716 3229.9,-72.4578 3238.66,-76.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3237.34,-79.3846 3247.92,-80.0341 3240.06,-72.9318 3237.34,-79.3846\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge42\" class=\"edge\"><title>layer3_block0_conv3&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-117.599C3220.66,-114.035 3229.9,-110.043 3238.66,-106.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3240.13,-109.44 3247.92,-102.262 3237.35,-103.013 3240.13,-109.44\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1 -->\n",
       "<g id=\"node39\" class=\"node\"><title>layer3_block1_bn1</title>\n",
       "<g id=\"a_node39\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3380,-109C3380,-109 3350,-109 3350,-109 3344,-109 3338,-103 3338,-97 3338,-97 3338,-85 3338,-85 3338,-79 3344,-73 3350,-73 3350,-73 3380,-73 3380,-73 3386,-73 3392,-79 3392,-85 3392,-85 3392,-97 3392,-97 3392,-103 3386,-109 3380,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3365\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_add&#45;&gt;layer3_block1_bn1 -->\n",
       "<g id=\"edge44\" class=\"edge\"><title>layer3_block0_add&#45;&gt;layer3_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3302.4,-91C3310.39,-91 3319.31,-91 3327.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3327.92,-94.5001 3337.92,-91 3327.92,-87.5001 3327.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1 -->\n",
       "<g id=\"node40\" class=\"node\"><title>layer3_block1_relu1</title>\n",
       "<g id=\"a_node40\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3470,-109C3470,-109 3440,-109 3440,-109 3434,-109 3428,-103 3428,-97 3428,-97 3428,-85 3428,-85 3428,-79 3434,-73 3440,-73 3440,-73 3470,-73 3470,-73 3476,-73 3482,-79 3482,-85 3482,-85 3482,-97 3482,-97 3482,-103 3476,-109 3470,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3455\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1&#45;&gt;layer3_block1_relu1 -->\n",
       "<g id=\"edge45\" class=\"edge\"><title>layer3_block1_bn1&#45;&gt;layer3_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3392.4,-91C3400.39,-91 3409.31,-91 3417.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3417.92,-94.5001 3427.92,-91 3417.92,-87.5001 3417.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1 -->\n",
       "<g id=\"node41\" class=\"node\"><title>layer3_block1_branch_conv1</title>\n",
       "<g id=\"a_node41\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3560,-89C3560,-89 3530,-89 3530,-89 3524,-89 3518,-83 3518,-77 3518,-77 3518,-65 3518,-65 3518,-59 3524,-53 3530,-53 3530,-53 3560,-53 3560,-53 3566,-53 3572,-59 3572,-65 3572,-65 3572,-77 3572,-77 3572,-83 3566,-89 3560,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3545\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1 -->\n",
       "<g id=\"edge46\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3482.4,-84.9993C3490.48,-83.1634 3499.51,-81.1121 3508.1,-79.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3508.94,-82.5568 3517.92,-76.9275 3507.39,-75.7308 3508.94,-82.5568\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_add -->\n",
       "<g id=\"node45\" class=\"node\"><title>layer3_block1_add</title>\n",
       "<g id=\"a_node45\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3920,-109C3920,-109 3890,-109 3890,-109 3884,-109 3878,-103 3878,-97 3878,-97 3878,-85 3878,-85 3878,-79 3884,-73 3890,-73 3890,-73 3920,-73 3920,-73 3926,-73 3932,-79 3932,-85 3932,-85 3932,-97 3932,-97 3932,-103 3926,-109 3920,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3905\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge50\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3479.74,-109.102C3488.74,-114.964 3499.38,-120.78 3510,-124 3562.78,-140.006 3578.84,-129 3634,-129 3634,-129 3634,-129 3726,-129 3781.16,-129 3797.22,-140.006 3850,-124 3857.3,-121.786 3864.61,-118.345 3871.4,-114.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3873.55,-117.295 3880.26,-109.102 3869.9,-111.318 3873.55,-117.295\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2 -->\n",
       "<g id=\"node42\" class=\"node\"><title>layer3_block1_branch_bn2</title>\n",
       "<g id=\"a_node42\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3650,-89C3650,-89 3620,-89 3620,-89 3614,-89 3608,-83 3608,-77 3608,-77 3608,-65 3608,-65 3608,-59 3614,-53 3620,-53 3620,-53 3650,-53 3650,-53 3656,-53 3662,-59 3662,-65 3662,-65 3662,-77 3662,-77 3662,-83 3656,-89 3650,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2 -->\n",
       "<g id=\"edge47\" class=\"edge\"><title>layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3572.4,-71C3580.39,-71 3589.31,-71 3597.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3597.92,-74.5001 3607.92,-71 3597.92,-67.5001 3597.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2 -->\n",
       "<g id=\"node43\" class=\"node\"><title>layer3_block1_branch_relu2</title>\n",
       "<g id=\"a_node43\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3740,-89C3740,-89 3710,-89 3710,-89 3704,-89 3698,-83 3698,-77 3698,-77 3698,-65 3698,-65 3698,-59 3704,-53 3710,-53 3710,-53 3740,-53 3740,-53 3746,-53 3752,-59 3752,-65 3752,-65 3752,-77 3752,-77 3752,-83 3746,-89 3740,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3725\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2 -->\n",
       "<g id=\"edge48\" class=\"edge\"><title>layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3662.4,-71C3670.39,-71 3679.31,-71 3687.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3687.92,-74.5001 3697.92,-71 3687.92,-67.5001 3687.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2 -->\n",
       "<g id=\"node44\" class=\"node\"><title>layer3_block1_branch_conv2</title>\n",
       "<g id=\"a_node44\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3830,-89C3830,-89 3800,-89 3800,-89 3794,-89 3788,-83 3788,-77 3788,-77 3788,-65 3788,-65 3788,-59 3794,-53 3800,-53 3800,-53 3830,-53 3830,-53 3836,-53 3842,-59 3842,-65 3842,-65 3842,-77 3842,-77 3842,-83 3836,-89 3830,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3815\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2 -->\n",
       "<g id=\"edge49\" class=\"edge\"><title>layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3752.4,-71C3760.39,-71 3769.31,-71 3777.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3777.92,-74.5001 3787.92,-71 3777.92,-67.5001 3777.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge51\" class=\"edge\"><title>layer3_block1_branch_conv2&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3842.4,-77.0007C3850.48,-78.8366 3859.51,-80.8879 3868.1,-82.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3867.39,-86.2692 3877.92,-85.0725 3868.94,-79.4432 3867.39,-86.2692\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1 -->\n",
       "<g id=\"node46\" class=\"node\"><title>layer4_block0_bn1</title>\n",
       "<g id=\"a_node46\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4014,-109C4014,-109 3984,-109 3984,-109 3978,-109 3972,-103 3972,-97 3972,-97 3972,-85 3972,-85 3972,-79 3978,-73 3984,-73 3984,-73 4014,-73 4014,-73 4020,-73 4026,-79 4026,-85 4026,-85 4026,-97 4026,-97 4026,-103 4020,-109 4014,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3999\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_add&#45;&gt;layer4_block0_bn1 -->\n",
       "<g id=\"edge52\" class=\"edge\"><title>layer3_block1_add&#45;&gt;layer4_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3932.12,-91C3941.34,-91 3951.89,-91 3961.8,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3961.82,-94.5001 3971.82,-91 3961.82,-87.5001 3961.82,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1 -->\n",
       "<g id=\"node47\" class=\"node\"><title>layer4_block0_relu1</title>\n",
       "<g id=\"a_node47\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4104,-109C4104,-109 4074,-109 4074,-109 4068,-109 4062,-103 4062,-97 4062,-97 4062,-85 4062,-85 4062,-79 4068,-73 4074,-73 4074,-73 4104,-73 4104,-73 4110,-73 4116,-79 4116,-85 4116,-85 4116,-97 4116,-97 4116,-103 4110,-109 4104,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"4089\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1&#45;&gt;layer4_block0_relu1 -->\n",
       "<g id=\"edge53\" class=\"edge\"><title>layer4_block0_bn1&#45;&gt;layer4_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4026.4,-91C4034.39,-91 4043.31,-91 4051.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4051.92,-94.5001 4061.92,-91 4051.92,-87.5001 4051.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1 -->\n",
       "<g id=\"node48\" class=\"node\"><title>layer4_block0_branch_conv1</title>\n",
       "<g id=\"a_node48\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4194,-68C4194,-68 4164,-68 4164,-68 4158,-68 4152,-62 4152,-56 4152,-56 4152,-44 4152,-44 4152,-38 4158,-32 4164,-32 4164,-32 4194,-32 4194,-32 4200,-32 4206,-38 4206,-44 4206,-44 4206,-56 4206,-56 4206,-62 4200,-68 4194,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4179\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1 -->\n",
       "<g id=\"edge54\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.4,-78.6986C4124.66,-74.8531 4133.9,-70.5468 4142.66,-66.463\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4144.33,-69.5472 4151.92,-62.1514 4141.38,-63.2021 4144.33,-69.5472\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3 -->\n",
       "<g id=\"node52\" class=\"node\"><title>layer4_block0_conv3</title>\n",
       "<g id=\"a_node52\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-143C4464,-143 4434,-143 4434,-143 4428,-143 4422,-137 4422,-131 4422,-131 4422,-119 4422,-119 4422,-113 4428,-107 4434,-107 4434,-107 4464,-107 4464,-107 4470,-107 4476,-113 4476,-119 4476,-119 4476,-131 4476,-131 4476,-137 4470,-143 4464,-143\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-121.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_conv3 -->\n",
       "<g id=\"edge58\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.22,-97.6371C4125,-99.6421 4134.88,-101.664 4144,-103 4239.91,-117.051 4354.23,-122.258 4411.49,-124.083\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.73,-127.592 4421.83,-124.395 4411.94,-120.595 4411.73,-127.592\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2 -->\n",
       "<g id=\"node49\" class=\"node\"><title>layer4_block0_branch_bn2</title>\n",
       "<g id=\"a_node49\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4284,-68C4284,-68 4254,-68 4254,-68 4248,-68 4242,-62 4242,-56 4242,-56 4242,-44 4242,-44 4242,-38 4248,-32 4254,-32 4254,-32 4284,-32 4284,-32 4290,-32 4296,-38 4296,-44 4296,-44 4296,-56 4296,-56 4296,-62 4290,-68 4284,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2 -->\n",
       "<g id=\"edge55\" class=\"edge\"><title>layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4206.4,-50C4214.39,-50 4223.31,-50 4231.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4231.92,-53.5001 4241.92,-50 4231.92,-46.5001 4231.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2 -->\n",
       "<g id=\"node50\" class=\"node\"><title>layer4_block0_branch_relu2</title>\n",
       "<g id=\"a_node50\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4374,-68C4374,-68 4344,-68 4344,-68 4338,-68 4332,-62 4332,-56 4332,-56 4332,-44 4332,-44 4332,-38 4338,-32 4344,-32 4344,-32 4374,-32 4374,-32 4380,-32 4386,-38 4386,-44 4386,-44 4386,-56 4386,-56 4386,-62 4380,-68 4374,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4359\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2 -->\n",
       "<g id=\"edge56\" class=\"edge\"><title>layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4296.4,-50C4304.39,-50 4313.31,-50 4321.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4321.92,-53.5001 4331.92,-50 4321.92,-46.5001 4321.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2 -->\n",
       "<g id=\"node51\" class=\"node\"><title>layer4_block0_branch_conv2</title>\n",
       "<g id=\"a_node51\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-68C4464,-68 4434,-68 4434,-68 4428,-68 4422,-62 4422,-56 4422,-56 4422,-44 4422,-44 4422,-38 4428,-32 4434,-32 4434,-32 4464,-32 4464,-32 4470,-32 4476,-38 4476,-44 4476,-44 4476,-56 4476,-56 4476,-62 4470,-68 4464,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2 -->\n",
       "<g id=\"edge57\" class=\"edge\"><title>layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4386.4,-50C4394.39,-50 4403.31,-50 4411.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.92,-53.5001 4421.92,-50 4411.92,-46.5001 4411.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_add -->\n",
       "<g id=\"node53\" class=\"node\"><title>layer4_block0_add</title>\n",
       "<g id=\"a_node53\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M4554,-105C4554,-105 4524,-105 4524,-105 4518,-105 4512,-99 4512,-93 4512,-93 4512,-81 4512,-81 4512,-75 4518,-69 4524,-69 4524,-69 4554,-69 4554,-69 4560,-69 4566,-75 4566,-81 4566,-81 4566,-93 4566,-93 4566,-99 4560,-105 4554,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4539\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge60\" class=\"edge\"><title>layer4_block0_branch_conv2&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-61.1012C4484.66,-64.5716 4493.9,-68.4578 4502.66,-72.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4501.34,-75.3846 4511.92,-76.0341 4504.06,-68.9318 4501.34,-75.3846\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge59\" class=\"edge\"><title>layer4_block0_conv3&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-113.599C4484.66,-110.035 4493.9,-106.043 4502.66,-102.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4504.13,-105.44 4511.92,-98.2623 4501.35,-99.0135 4504.13,-105.44\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1 -->\n",
       "<g id=\"node54\" class=\"node\"><title>layer4_block1_bn1</title>\n",
       "<g id=\"a_node54\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4644,-105C4644,-105 4614,-105 4614,-105 4608,-105 4602,-99 4602,-93 4602,-93 4602,-81 4602,-81 4602,-75 4608,-69 4614,-69 4614,-69 4644,-69 4644,-69 4650,-69 4656,-75 4656,-81 4656,-81 4656,-93 4656,-93 4656,-99 4650,-105 4644,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4629\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_add&#45;&gt;layer4_block1_bn1 -->\n",
       "<g id=\"edge61\" class=\"edge\"><title>layer4_block0_add&#45;&gt;layer4_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4566.4,-87C4574.39,-87 4583.31,-87 4591.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4591.92,-90.5001 4601.92,-87 4591.92,-83.5001 4591.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1 -->\n",
       "<g id=\"node55\" class=\"node\"><title>layer4_block1_relu1</title>\n",
       "<g id=\"a_node55\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4734,-105C4734,-105 4704,-105 4704,-105 4698,-105 4692,-99 4692,-93 4692,-93 4692,-81 4692,-81 4692,-75 4698,-69 4704,-69 4704,-69 4734,-69 4734,-69 4740,-69 4746,-75 4746,-81 4746,-81 4746,-93 4746,-93 4746,-99 4740,-105 4734,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4719\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1&#45;&gt;layer4_block1_relu1 -->\n",
       "<g id=\"edge62\" class=\"edge\"><title>layer4_block1_bn1&#45;&gt;layer4_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4656.4,-87C4664.39,-87 4673.31,-87 4681.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4681.92,-90.5001 4691.92,-87 4681.92,-83.5001 4681.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1 -->\n",
       "<g id=\"node56\" class=\"node\"><title>layer4_block1_branch_conv1</title>\n",
       "<g id=\"a_node56\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4824,-85C4824,-85 4794,-85 4794,-85 4788,-85 4782,-79 4782,-73 4782,-73 4782,-61 4782,-61 4782,-55 4788,-49 4794,-49 4794,-49 4824,-49 4824,-49 4830,-49 4836,-55 4836,-61 4836,-61 4836,-73 4836,-73 4836,-79 4830,-85 4824,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4809\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1 -->\n",
       "<g id=\"edge63\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4746.4,-80.9993C4754.48,-79.1634 4763.51,-77.1121 4772.1,-75.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4772.94,-78.5568 4781.92,-72.9275 4771.39,-71.7308 4772.94,-78.5568\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_add -->\n",
       "<g id=\"node60\" class=\"node\"><title>layer4_block1_add</title>\n",
       "<g id=\"a_node60\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M5184,-105C5184,-105 5154,-105 5154,-105 5148,-105 5142,-99 5142,-93 5142,-93 5142,-81 5142,-81 5142,-75 5148,-69 5154,-69 5154,-69 5184,-69 5184,-69 5190,-69 5196,-75 5196,-81 5196,-81 5196,-93 5196,-93 5196,-99 5190,-105 5184,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5169\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge67\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4743.74,-105.102C4752.74,-110.964 4763.38,-116.78 4774,-120 4826.78,-136.006 4842.84,-125 4898,-125 4898,-125 4898,-125 4990,-125 5045.16,-125 5061.22,-136.006 5114,-120 5121.3,-117.786 5128.61,-114.345 5135.4,-110.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5137.55,-113.295 5144.26,-105.102 5133.9,-107.318 5137.55,-113.295\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2 -->\n",
       "<g id=\"node57\" class=\"node\"><title>layer4_block1_branch_bn2</title>\n",
       "<g id=\"a_node57\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4914,-85C4914,-85 4884,-85 4884,-85 4878,-85 4872,-79 4872,-73 4872,-73 4872,-61 4872,-61 4872,-55 4878,-49 4884,-49 4884,-49 4914,-49 4914,-49 4920,-49 4926,-55 4926,-61 4926,-61 4926,-73 4926,-73 4926,-79 4920,-85 4914,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2 -->\n",
       "<g id=\"edge64\" class=\"edge\"><title>layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4836.4,-67C4844.39,-67 4853.31,-67 4861.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4861.92,-70.5001 4871.92,-67 4861.92,-63.5001 4861.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2 -->\n",
       "<g id=\"node58\" class=\"node\"><title>layer4_block1_branch_relu2</title>\n",
       "<g id=\"a_node58\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M5004,-85C5004,-85 4974,-85 4974,-85 4968,-85 4962,-79 4962,-73 4962,-73 4962,-61 4962,-61 4962,-55 4968,-49 4974,-49 4974,-49 5004,-49 5004,-49 5010,-49 5016,-55 5016,-61 5016,-61 5016,-73 5016,-73 5016,-79 5010,-85 5004,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4989\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2 -->\n",
       "<g id=\"edge65\" class=\"edge\"><title>layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4926.4,-67C4934.39,-67 4943.31,-67 4951.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4951.92,-70.5001 4961.92,-67 4951.92,-63.5001 4951.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2 -->\n",
       "<g id=\"node59\" class=\"node\"><title>layer4_block1_branch_conv2</title>\n",
       "<g id=\"a_node59\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M5094,-85C5094,-85 5064,-85 5064,-85 5058,-85 5052,-79 5052,-73 5052,-73 5052,-61 5052,-61 5052,-55 5058,-49 5064,-49 5064,-49 5094,-49 5094,-49 5100,-49 5106,-55 5106,-61 5106,-61 5106,-73 5106,-73 5106,-79 5100,-85 5094,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"5079\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2 -->\n",
       "<g id=\"edge66\" class=\"edge\"><title>layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5016.4,-67C5024.39,-67 5033.31,-67 5041.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5041.92,-70.5001 5051.92,-67 5041.92,-63.5001 5041.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge68\" class=\"edge\"><title>layer4_block1_branch_conv2&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5106.4,-73.0007C5114.48,-74.8366 5123.51,-76.8879 5132.1,-78.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5131.39,-82.2692 5141.92,-81.0725 5132.94,-75.4432 5131.39,-82.2692\"/>\n",
       "</g>\n",
       "<!-- final_in -->\n",
       "<g id=\"node61\" class=\"node\"><title>final_in</title>\n",
       "<g id=\"a_node61\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5274,-105C5274,-105 5244,-105 5244,-105 5238,-105 5232,-99 5232,-93 5232,-93 5232,-81 5232,-81 5232,-75 5238,-69 5244,-69 5244,-69 5274,-69 5274,-69 5280,-69 5286,-75 5286,-81 5286,-81 5286,-93 5286,-93 5286,-99 5280,-105 5274,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5259\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">in</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_add&#45;&gt;final_in -->\n",
       "<g id=\"edge69\" class=\"edge\"><title>layer4_block1_add&#45;&gt;final_in</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5196.4,-87C5204.39,-87 5213.31,-87 5221.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5221.92,-90.5001 5231.92,-87 5221.92,-83.5001 5221.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_maxpool -->\n",
       "<g id=\"node62\" class=\"node\"><title>final_maxpool</title>\n",
       "<g id=\"a_node62\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#fdb462\" stroke=\"black\" d=\"M5375,-78C5375,-78 5334,-78 5334,-78 5328,-78 5322,-72 5322,-66 5322,-66 5322,-54 5322,-54 5322,-48 5328,-42 5334,-42 5334,-42 5375,-42 5375,-42 5381,-42 5387,-48 5387,-54 5387,-54 5387,-66 5387,-66 5387,-72 5381,-78 5375,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">maxpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_maxpool -->\n",
       "<g id=\"edge70\" class=\"edge\"><title>final_in&#45;&gt;final_maxpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-79.479C5294.13,-77.1436 5303.26,-74.509 5312.12,-71.9497\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.31,-75.2492 5321.94,-69.1121 5311.37,-68.524 5313.31,-75.2492\"/>\n",
       "</g>\n",
       "<!-- final_avgpool -->\n",
       "<g id=\"node63\" class=\"node\"><title>final_avgpool</title>\n",
       "<g id=\"a_node63\"><a xlink:title=\"AvgPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;ceil_mode&#39;: False, &#39;count_include_pad&#39;: True}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M5372.5,-132C5372.5,-132 5336.5,-132 5336.5,-132 5330.5,-132 5324.5,-126 5324.5,-120 5324.5,-120 5324.5,-108 5324.5,-108 5324.5,-102 5330.5,-96 5336.5,-96 5336.5,-96 5372.5,-96 5372.5,-96 5378.5,-96 5384.5,-102 5384.5,-108 5384.5,-108 5384.5,-120 5384.5,-120 5384.5,-126 5378.5,-132 5372.5,-132\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-110.3\" font-family=\"Times,serif\" font-size=\"14.00\">avgpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_avgpool -->\n",
       "<g id=\"edge71\" class=\"edge\"><title>final_in&#45;&gt;final_avgpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-94.521C5294.87,-97.0688 5304.92,-99.9726 5314.53,-102.746\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.73,-106.157 5324.3,-105.569 5315.67,-99.4322 5313.73,-106.157\"/>\n",
       "</g>\n",
       "<!-- final_concat -->\n",
       "<g id=\"node64\" class=\"node\"><title>final_concat</title>\n",
       "<g id=\"a_node64\"><a xlink:title=\"Concat {}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M5465,-105C5465,-105 5435,-105 5435,-105 5429,-105 5423,-99 5423,-93 5423,-93 5423,-81 5423,-81 5423,-75 5429,-69 5435,-69 5435,-69 5465,-69 5465,-69 5471,-69 5477,-75 5477,-81 5477,-81 5477,-93 5477,-93 5477,-99 5471,-105 5465,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5450\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">concat</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_maxpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge72\" class=\"edge\"><title>final_maxpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5387.15,-69.141C5395.53,-71.5601 5404.63,-74.1884 5413.21,-76.6659\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5412.39,-80.0717 5422.97,-79.4835 5414.33,-73.3465 5412.39,-80.0717\"/>\n",
       "</g>\n",
       "<!-- final_avgpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge73\" class=\"edge\"><title>final_avgpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5384.57,-105.605C5393.55,-103.013 5403.54,-100.127 5412.93,-97.4155\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5414.17,-100.702 5422.8,-94.5648 5412.22,-93.9766 5414.17,-100.702\"/>\n",
       "</g>\n",
       "<!-- final_flatten -->\n",
       "<g id=\"node65\" class=\"node\"><title>final_flatten</title>\n",
       "<g id=\"a_node65\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M5555,-105C5555,-105 5525,-105 5525,-105 5519,-105 5513,-99 5513,-93 5513,-93 5513,-81 5513,-81 5513,-75 5519,-69 5525,-69 5525,-69 5555,-69 5555,-69 5561,-69 5567,-75 5567,-81 5567,-81 5567,-93 5567,-93 5567,-99 5561,-105 5555,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5540\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_concat&#45;&gt;final_flatten -->\n",
       "<g id=\"edge74\" class=\"edge\"><title>final_concat&#45;&gt;final_flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5477.4,-87C5485.39,-87 5494.31,-87 5502.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5502.92,-90.5001 5512.92,-87 5502.92,-83.5001 5502.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_linear -->\n",
       "<g id=\"node66\" class=\"node\"><title>final_linear</title>\n",
       "<g id=\"a_node66\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: Parameter containing:\\ntensor([ 0.0179, &#45;0.0001, &#160;0.0160, &#160;0.0307, &#45;0.0357, &#45;0.0042, &#160;0.0234, &#160;0.0428,\\n &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#45;0.0372, &#45;0.0280], requires_grad=True)}\">\n",
       "<path fill=\"#ccebc5\" stroke=\"black\" d=\"M5645,-105C5645,-105 5615,-105 5615,-105 5609,-105 5603,-99 5603,-93 5603,-93 5603,-81 5603,-81 5603,-75 5609,-69 5615,-69 5615,-69 5645,-69 5645,-69 5651,-69 5657,-75 5657,-81 5657,-81 5657,-93 5657,-93 5657,-99 5651,-105 5645,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5630\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_flatten&#45;&gt;final_linear -->\n",
       "<g id=\"edge75\" class=\"edge\"><title>final_flatten&#45;&gt;final_linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5567.4,-87C5575.39,-87 5584.31,-87 5592.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5592.92,-90.5001 5602.92,-87 5592.92,-83.5001 5592.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node67\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node67\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5746,-105C5746,-105 5705,-105 5705,-105 5699,-105 5693,-99 5693,-93 5693,-93 5693,-81 5693,-81 5693,-75 5699,-69 5705,-69 5705,-69 5746,-69 5746,-69 5752,-69 5758,-75 5758,-81 5758,-81 5758,-93 5758,-93 5758,-99 5752,-105 5746,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5725.5\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_linear&#45;&gt;classifier -->\n",
       "<g id=\"edge76\" class=\"edge\"><title>final_linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5657.04,-87C5665.05,-87 5674.06,-87 5682.83,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5682.95,-90.5001 5692.94,-87 5682.94,-83.5001 5682.95,-90.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node68\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-96C42,-96 12,-96 12,-96 6,-96 0,-90 0,-84 0,-84 0,-72 0,-72 0,-66 6,-60 12,-60 12,-60 42,-60 42,-60 48,-60 54,-66 54,-72 54,-72 54,-84 54,-84 54,-90 48,-96 42,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-78C62.3932,-78 71.3106,-78 79.8241,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-81.5001 89.919,-78 79.919,-74.5001 79.919,-81.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7f0100e63320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.0067       6.8526       1.6560       0.3829       0.4870       1.3605       0.5198       7.3396\n",
      "           2       0.0133       6.7992       1.0044       0.6424       0.4867       0.8560       0.7007      14.6254\n",
      "           3       0.0200       6.7301       0.7538       0.7354       0.4526       0.7101       0.7552      21.8081\n",
      "           4       0.0267       6.7953       0.6382       0.7780       0.4643       0.5976       0.8000      29.0677\n",
      "           5       0.0333       6.7002       0.5669       0.8043       0.4491       0.7872       0.7472      36.2170\n",
      "           6       0.0400       6.6877       0.5093       0.8242       0.4683       0.5423       0.8148      43.3730\n",
      "           7       0.0467       6.6392       0.4694       0.8400       0.4509       0.5086       0.8232      50.4631\n",
      "           8       0.0533       6.8832       0.4376       0.8495       0.5065       0.6590       0.7789      57.8527\n",
      "           9       0.0600       6.7262       0.4148       0.8567       0.5136       0.4793       0.8347      65.0925\n",
      "          10       0.0667       6.7130       0.3970       0.8634       0.4580       0.4796       0.8415      72.2634\n",
      "          11       0.0733       6.7245       0.3905       0.8659       0.4720       0.4904       0.8363      79.4599\n",
      "          12       0.0800       6.7151       0.3910       0.8660       0.4569       0.5269       0.8256      86.6319\n",
      "          13       0.0867       6.6864       0.3906       0.8653       0.4674       0.5121       0.8281      93.7858\n",
      "          14       0.0933       6.8104       0.3871       0.8671       0.4879       0.5360       0.8207     101.0841\n",
      "          15       0.1000       6.8254       0.3927       0.8661       0.4534       0.4881       0.8406     108.3629\n",
      "          16       0.0937       6.6754       0.3791       0.8707       0.4758       0.5650       0.8128     115.5141\n",
      "          17       0.0873       6.7389       0.3591       0.8772       0.5311       0.4912       0.8358     122.7842\n",
      "          18       0.0810       7.0205       0.3387       0.8831       0.5838       0.5871       0.8079     130.3885\n",
      "          19       0.0747       6.7949       0.3168       0.8909       0.4509       0.4533       0.8473     137.6343\n",
      "          20       0.0683       6.6850       0.2996       0.8970       0.4737       0.4287       0.8520     144.7930\n",
      "          21       0.0620       6.7891       0.2821       0.9021       0.4626       0.3751       0.8756     152.0448\n",
      "          22       0.0557       6.7628       0.2640       0.9082       0.4890       0.3850       0.8717     159.2965\n",
      "          23       0.0493       6.8803       0.2406       0.9169       0.4866       0.4402       0.8652     166.6635\n",
      "          24       0.0430       6.7134       0.2198       0.9229       0.4975       0.3715       0.8812     173.8744\n",
      "          25       0.0367       6.7090       0.1988       0.9306       0.4827       0.2791       0.9072     181.0661\n",
      "          26       0.0303       6.6691       0.1706       0.9409       0.4428       0.3263       0.8943     188.1781\n",
      "          27       0.0240       6.7302       0.1450       0.9502       0.4633       0.2830       0.9118     195.3716\n",
      "          28       0.0177       6.6754       0.1145       0.9607       0.4754       0.2588       0.9239     202.5225\n",
      "          29       0.0113       6.7495       0.0854       0.9697       0.4495       0.2262       0.9302     209.7214\n",
      "          30       0.0050       6.7638       0.0585       0.9808       0.4848       0.2081       0.9377     216.9701\n",
      "          31       0.0040       6.7413       0.0434       0.9864       0.4763       0.2087       0.9379     224.1877\n",
      "          32       0.0030       6.6756       0.0363       0.9892       0.4653       0.2048       0.9391     231.3286\n",
      "          33       0.0020       6.6711       0.0325       0.9905       0.4918       0.2047       0.9408     238.4914\n",
      "          34       0.0010       6.7179       0.0289       0.9917       0.4553       0.2035       0.9396     245.6647\n",
      "          35       0.0000       6.7006       0.0287       0.9919       0.5021       0.2045       0.9398     252.8674\n"
     ]
    }
   ],
   "source": [
    "lr_schedule = PiecewiseLinear([0, 15, 30, 35], [0, 0.1, 0.005, 0])\n",
    "batch_size = 128\n",
    "\n",
    "n = DAWN_net()\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device)\n",
    "#convert all children including batch norms to half precision (triggering slow codepath!)\n",
    "for v in model.children(): \n",
    "    v.half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR()])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 1: Baseline](https://www.myrtle.ai/2018/09/24/how_to_train_your_resnet_1/) - 0 dataloader processes (297s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"43pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 43.08\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.187305 0.187305) rotate(0) translate(4 226)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-226 5762,-226 5762,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-52C94,-52 140,-52 140,-52 146,-52 152,-58 152,-64 152,-64 152,-115 152,-115 152,-121 146,-127 140,-127 140,-127 94,-127 94,-127 88,-127 82,-121 82,-115 82,-115 82,-64 82,-64 82,-58 88,-52 94,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M176,-16C176,-16 1408,-16 1408,-16 1414,-16 1420,-22 1420,-28 1420,-28 1420,-167 1420,-167 1420,-173 1414,-179 1408,-179 1408,-179 176,-179 176,-179 170,-179 164,-173 164,-167 164,-167 164,-28 164,-28 164,-22 170,-16 176,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"792\" y=\"-163.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer1_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M184,-24C184,-24 770,-24 770,-24 776,-24 782,-30 782,-36 782,-36 782,-136 782,-136 782,-142 776,-148 770,-148 770,-148 184,-148 184,-148 178,-148 172,-142 172,-136 172,-136 172,-36 172,-36 172,-30 178,-24 184,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer1_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-32C364,-32 680,-32 680,-32 686,-32 692,-38 692,-44 692,-44 692,-95 692,-95 692,-101 686,-107 680,-107 680,-107 364,-107 364,-107 358,-107 352,-101 352,-95 352,-95 352,-44 352,-44 352,-38 358,-32 364,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer1_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M814,-24C814,-24 1400,-24 1400,-24 1406,-24 1412,-30 1412,-36 1412,-36 1412,-136 1412,-136 1412,-142 1406,-148 1400,-148 1400,-148 814,-148 814,-148 808,-148 802,-142 802,-136 802,-136 802,-36 802,-36 802,-30 808,-24 814,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer1_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M994,-32C994,-32 1310,-32 1310,-32 1316,-32 1322,-38 1322,-44 1322,-44 1322,-95 1322,-95 1322,-101 1316,-107 1310,-107 1310,-107 994,-107 994,-107 988,-107 982,-101 982,-95 982,-95 982,-44 982,-44 982,-38 988,-32 994,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"1152\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1440,-17C1440,-17 2672,-17 2672,-17 2678,-17 2684,-23 2684,-29 2684,-29 2684,-202 2684,-202 2684,-208 2678,-214 2672,-214 2672,-214 1440,-214 1440,-214 1434,-214 1428,-208 1428,-202 1428,-202 1428,-29 1428,-29 1428,-23 1434,-17 1440,-17\"/>\n",
       "<text text-anchor=\"middle\" x=\"2056\" y=\"-198.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_layer2_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1448,-25C1448,-25 2034,-25 2034,-25 2040,-25 2046,-31 2046,-37 2046,-37 2046,-171 2046,-171 2046,-177 2040,-183 2034,-183 2034,-183 1448,-183 1448,-183 1442,-183 1436,-177 1436,-171 1436,-171 1436,-37 1436,-37 1436,-31 1442,-25 1448,-25\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-167.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_layer2_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1628,-33C1628,-33 1944,-33 1944,-33 1950,-33 1956,-39 1956,-45 1956,-45 1956,-96 1956,-96 1956,-102 1950,-108 1944,-108 1944,-108 1628,-108 1628,-108 1622,-108 1616,-102 1616,-96 1616,-96 1616,-45 1616,-45 1616,-39 1622,-33 1628,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"1786\" y=\"-92.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_layer2_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2078,-30C2078,-30 2664,-30 2664,-30 2670,-30 2676,-36 2676,-42 2676,-42 2676,-142 2676,-142 2676,-148 2670,-154 2664,-154 2664,-154 2078,-154 2078,-154 2072,-154 2066,-148 2066,-142 2066,-142 2066,-42 2066,-42 2066,-36 2072,-30 2078,-30\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-138.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust11\" class=\"cluster\"><title>cluster_layer2_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2258,-38C2258,-38 2574,-38 2574,-38 2580,-38 2586,-44 2586,-50 2586,-50 2586,-101 2586,-101 2586,-107 2580,-113 2574,-113 2574,-113 2258,-113 2258,-113 2252,-113 2246,-107 2246,-101 2246,-101 2246,-50 2246,-50 2246,-44 2252,-38 2258,-38\"/>\n",
       "<text text-anchor=\"middle\" x=\"2416\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust12\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2704,-12C2704,-12 3936,-12 3936,-12 3942,-12 3948,-18 3948,-24 3948,-24 3948,-197 3948,-197 3948,-203 3942,-209 3936,-209 3936,-209 2704,-209 2704,-209 2698,-209 2692,-203 2692,-197 2692,-197 2692,-24 2692,-24 2692,-18 2698,-12 2704,-12\"/>\n",
       "<text text-anchor=\"middle\" x=\"3320\" y=\"-193.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust13\" class=\"cluster\"><title>cluster_layer3_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2712,-20C2712,-20 3298,-20 3298,-20 3304,-20 3310,-26 3310,-32 3310,-32 3310,-166 3310,-166 3310,-172 3304,-178 3298,-178 3298,-178 2712,-178 2712,-178 2706,-178 2700,-172 2700,-166 2700,-166 2700,-32 2700,-32 2700,-26 2706,-20 2712,-20\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-162.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust14\" class=\"cluster\"><title>cluster_layer3_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2892,-28C2892,-28 3208,-28 3208,-28 3214,-28 3220,-34 3220,-40 3220,-40 3220,-91 3220,-91 3220,-97 3214,-103 3208,-103 3208,-103 2892,-103 2892,-103 2886,-103 2880,-97 2880,-91 2880,-91 2880,-40 2880,-40 2880,-34 2886,-28 2892,-28\"/>\n",
       "<text text-anchor=\"middle\" x=\"3050\" y=\"-87.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust15\" class=\"cluster\"><title>cluster_layer3_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3342,-37C3342,-37 3928,-37 3928,-37 3934,-37 3940,-43 3940,-49 3940,-49 3940,-149 3940,-149 3940,-155 3934,-161 3928,-161 3928,-161 3342,-161 3342,-161 3336,-161 3330,-155 3330,-149 3330,-149 3330,-49 3330,-49 3330,-43 3336,-37 3342,-37\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-145.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust16\" class=\"cluster\"><title>cluster_layer3_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3522,-45C3522,-45 3838,-45 3838,-45 3844,-45 3850,-51 3850,-57 3850,-57 3850,-108 3850,-108 3850,-114 3844,-120 3838,-120 3838,-120 3522,-120 3522,-120 3516,-120 3510,-114 3510,-108 3510,-108 3510,-57 3510,-57 3510,-51 3516,-45 3522,-45\"/>\n",
       "<text text-anchor=\"middle\" x=\"3680\" y=\"-104.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust17\" class=\"cluster\"><title>cluster_layer4</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3968,-8C3968,-8 5200,-8 5200,-8 5206,-8 5212,-14 5212,-20 5212,-20 5212,-193 5212,-193 5212,-199 5206,-205 5200,-205 5200,-205 3968,-205 3968,-205 3962,-205 3956,-199 3956,-193 3956,-193 3956,-20 3956,-20 3956,-14 3962,-8 3968,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"4584\" y=\"-189.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer4</text>\n",
       "</g>\n",
       "<g id=\"clust18\" class=\"cluster\"><title>cluster_layer4_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3976,-16C3976,-16 4562,-16 4562,-16 4568,-16 4574,-22 4574,-28 4574,-28 4574,-162 4574,-162 4574,-168 4568,-174 4562,-174 4562,-174 3976,-174 3976,-174 3970,-174 3964,-168 3964,-162 3964,-162 3964,-28 3964,-28 3964,-22 3970,-16 3976,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-158.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust19\" class=\"cluster\"><title>cluster_layer4_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4156,-24C4156,-24 4472,-24 4472,-24 4478,-24 4484,-30 4484,-36 4484,-36 4484,-87 4484,-87 4484,-93 4478,-99 4472,-99 4472,-99 4156,-99 4156,-99 4150,-99 4144,-93 4144,-87 4144,-87 4144,-36 4144,-36 4144,-30 4150,-24 4156,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"4314\" y=\"-83.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust20\" class=\"cluster\"><title>cluster_layer4_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4606,-33C4606,-33 5192,-33 5192,-33 5198,-33 5204,-39 5204,-45 5204,-45 5204,-145 5204,-145 5204,-151 5198,-157 5192,-157 5192,-157 4606,-157 4606,-157 4600,-157 4594,-151 4594,-145 4594,-145 4594,-45 4594,-45 4594,-39 4600,-33 4606,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-141.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust21\" class=\"cluster\"><title>cluster_layer4_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4786,-41C4786,-41 5102,-41 5102,-41 5108,-41 5114,-47 5114,-53 5114,-53 5114,-104 5114,-104 5114,-110 5108,-116 5102,-116 5102,-116 4786,-116 4786,-116 4780,-116 4774,-110 4774,-104 4774,-104 4774,-53 4774,-53 4774,-47 4780,-41 4786,-41\"/>\n",
       "<text text-anchor=\"middle\" x=\"4944\" y=\"-100.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust22\" class=\"cluster\"><title>cluster_final</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M5236,-34C5236,-34 5653,-34 5653,-34 5659,-34 5665,-40 5665,-46 5665,-46 5665,-151 5665,-151 5665,-157 5659,-163 5653,-163 5653,-163 5236,-163 5236,-163 5230,-163 5224,-157 5224,-151 5224,-151 5224,-46 5224,-46 5224,-40 5230,-34 5236,-34\"/>\n",
       "<text text-anchor=\"middle\" x=\"5444.5\" y=\"-147.8\" font-family=\"Times,serif\" font-size=\"14.00\">final</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-96C132,-96 102,-96 102,-96 96,-96 90,-90 90,-84 90,-84 90,-72 90,-72 90,-66 96,-60 102,-60 102,-60 132,-60 132,-60 138,-60 144,-66 144,-72 144,-72 144,-84 144,-84 144,-90 138,-96 132,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1 -->\n",
       "<g id=\"node2\" class=\"node\"><title>layer1_block0_bn1</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-96C222,-96 192,-96 192,-96 186,-96 180,-90 180,-84 180,-84 180,-72 180,-72 180,-66 186,-60 192,-60 192,-60 222,-60 222,-60 228,-60 234,-66 234,-72 234,-72 234,-84 234,-84 234,-90 228,-96 222,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;layer1_block0_bn1 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;layer1_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-78C152.393,-78 161.311,-78 169.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-81.5001 179.919,-78 169.919,-74.5001 169.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>layer1_block0_relu1</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-96C312,-96 282,-96 282,-96 276,-96 270,-90 270,-84 270,-84 270,-72 270,-72 270,-66 276,-60 282,-60 282,-60 312,-60 312,-60 318,-60 324,-66 324,-72 324,-72 324,-84 324,-84 324,-90 318,-96 312,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1&#45;&gt;layer1_block0_relu1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>layer1_block0_bn1&#45;&gt;layer1_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-78C242.393,-78 251.311,-78 259.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-81.5001 269.919,-78 259.919,-74.5001 259.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_block0_branch_conv1</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-76C402,-76 372,-76 372,-76 366,-76 360,-70 360,-64 360,-64 360,-52 360,-52 360,-46 366,-40 372,-40 372,-40 402,-40 402,-40 408,-40 414,-46 414,-52 414,-52 414,-64 414,-64 414,-70 408,-76 402,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-71.9993C332.481,-70.1634 341.507,-68.1121 350.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"350.943,-69.5568 359.919,-63.9275 349.392,-62.7308 350.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_add -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer1_block0_add</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M762,-96C762,-96 732,-96 732,-96 726,-96 720,-90 720,-84 720,-84 720,-72 720,-72 720,-66 726,-60 732,-60 732,-60 762,-60 762,-60 768,-60 774,-66 774,-72 774,-72 774,-84 774,-84 774,-90 768,-96 762,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"747\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M321.737,-96.1018C330.735,-101.964 341.381,-107.78 352,-111 404.782,-127.006 420.844,-116 476,-116 476,-116 476,-116 568,-116 623.156,-116 639.218,-127.006 692,-111 699.301,-108.786 706.614,-105.345 713.396,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"715.545,-104.295 722.263,-96.1018 711.902,-98.3176 715.545,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2 -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_block0_branch_bn2</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-76C492,-76 462,-76 462,-76 456,-76 450,-70 450,-64 450,-64 450,-52 450,-52 450,-46 456,-40 462,-40 462,-40 492,-40 492,-40 498,-40 504,-46 504,-52 504,-52 504,-64 504,-64 504,-70 498,-76 492,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-58C422.393,-58 431.311,-58 439.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-61.5001 449.919,-58 439.919,-54.5001 439.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2 -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer1_block0_branch_relu2</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-76C582,-76 552,-76 552,-76 546,-76 540,-70 540,-64 540,-64 540,-52 540,-52 540,-46 546,-40 552,-40 552,-40 582,-40 582,-40 588,-40 594,-46 594,-52 594,-52 594,-64 594,-64 594,-70 588,-76 582,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-58C512.393,-58 521.311,-58 529.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-61.5001 539.919,-58 529.919,-54.5001 529.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2 -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer1_block0_branch_conv2</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M672,-76C672,-76 642,-76 642,-76 636,-76 630,-70 630,-64 630,-64 630,-52 630,-52 630,-46 636,-40 642,-40 642,-40 672,-40 672,-40 678,-40 684,-46 684,-52 684,-52 684,-64 684,-64 684,-70 678,-76 672,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-58C602.393,-58 611.311,-58 619.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-61.5001 629.919,-58 619.919,-54.5001 619.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer1_block0_branch_conv2&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.403,-64.0007C692.481,-65.8366 701.507,-67.8879 710.105,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"709.392,-73.2692 719.919,-72.0725 710.943,-66.4432 709.392,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1 -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer1_block1_bn1</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M852,-96C852,-96 822,-96 822,-96 816,-96 810,-90 810,-84 810,-84 810,-72 810,-72 810,-66 816,-60 822,-60 822,-60 852,-60 852,-60 858,-60 864,-66 864,-72 864,-72 864,-84 864,-84 864,-90 858,-96 852,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_add&#45;&gt;layer1_block1_bn1 -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer1_block0_add&#45;&gt;layer1_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.403,-78C782.393,-78 791.311,-78 799.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"799.919,-81.5001 809.919,-78 799.919,-74.5001 799.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1 -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer1_block1_relu1</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M942,-96C942,-96 912,-96 912,-96 906,-96 900,-90 900,-84 900,-84 900,-72 900,-72 900,-66 906,-60 912,-60 912,-60 942,-60 942,-60 948,-60 954,-66 954,-72 954,-72 954,-84 954,-84 954,-90 948,-96 942,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1&#45;&gt;layer1_block1_relu1 -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer1_block1_bn1&#45;&gt;layer1_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.403,-78C872.393,-78 881.311,-78 889.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"889.919,-81.5001 899.919,-78 889.919,-74.5001 889.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1 -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer1_block1_branch_conv1</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1032,-76C1032,-76 1002,-76 1002,-76 996,-76 990,-70 990,-64 990,-64 990,-52 990,-52 990,-46 996,-40 1002,-40 1002,-40 1032,-40 1032,-40 1038,-40 1044,-46 1044,-52 1044,-52 1044,-64 1044,-64 1044,-70 1038,-76 1032,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1 -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M954.403,-71.9993C962.481,-70.1634 971.507,-68.1121 980.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"980.943,-69.5568 989.919,-63.9275 979.392,-62.7308 980.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_add -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer1_block1_add</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1392,-96C1392,-96 1362,-96 1362,-96 1356,-96 1350,-90 1350,-84 1350,-84 1350,-72 1350,-72 1350,-66 1356,-60 1362,-60 1362,-60 1392,-60 1392,-60 1398,-60 1404,-66 1404,-72 1404,-72 1404,-84 1404,-84 1404,-90 1398,-96 1392,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1377\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M951.737,-96.1018C960.735,-101.964 971.381,-107.78 982,-111 1034.78,-127.006 1050.84,-116 1106,-116 1106,-116 1106,-116 1198,-116 1253.16,-116 1269.22,-127.006 1322,-111 1329.3,-108.786 1336.61,-105.345 1343.4,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1345.55,-104.295 1352.26,-96.1018 1341.9,-98.3176 1345.55,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2 -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer1_block1_branch_bn2</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1122,-76C1122,-76 1092,-76 1092,-76 1086,-76 1080,-70 1080,-64 1080,-64 1080,-52 1080,-52 1080,-46 1086,-40 1092,-40 1092,-40 1122,-40 1122,-40 1128,-40 1134,-46 1134,-52 1134,-52 1134,-64 1134,-64 1134,-70 1128,-76 1122,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2 -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.4,-58C1052.39,-58 1061.31,-58 1069.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1069.92,-61.5001 1079.92,-58 1069.92,-54.5001 1069.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2 -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer1_block1_branch_relu2</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1212,-76C1212,-76 1182,-76 1182,-76 1176,-76 1170,-70 1170,-64 1170,-64 1170,-52 1170,-52 1170,-46 1176,-40 1182,-40 1182,-40 1212,-40 1212,-40 1218,-40 1224,-46 1224,-52 1224,-52 1224,-64 1224,-64 1224,-70 1218,-76 1212,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2 -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1134.4,-58C1142.39,-58 1151.31,-58 1159.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1159.92,-61.5001 1169.92,-58 1159.92,-54.5001 1159.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2 -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer1_block1_branch_conv2</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1302,-76C1302,-76 1272,-76 1272,-76 1266,-76 1260,-70 1260,-64 1260,-64 1260,-52 1260,-52 1260,-46 1266,-40 1272,-40 1272,-40 1302,-40 1302,-40 1308,-40 1314,-46 1314,-52 1314,-52 1314,-64 1314,-64 1314,-70 1308,-76 1302,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1287\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2 -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1224.4,-58C1232.39,-58 1241.31,-58 1249.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1249.92,-61.5001 1259.92,-58 1249.92,-54.5001 1249.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>layer1_block1_branch_conv2&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1314.4,-64.0007C1322.48,-65.8366 1331.51,-67.8879 1340.1,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1339.39,-73.2692 1349.92,-72.0725 1340.94,-66.4432 1339.39,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1 -->\n",
       "<g id=\"node16\" class=\"node\"><title>layer2_block0_bn1</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1486,-96C1486,-96 1456,-96 1456,-96 1450,-96 1444,-90 1444,-84 1444,-84 1444,-72 1444,-72 1444,-66 1450,-60 1456,-60 1456,-60 1486,-60 1486,-60 1492,-60 1498,-66 1498,-72 1498,-72 1498,-84 1498,-84 1498,-90 1492,-96 1486,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1471\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_add&#45;&gt;layer2_block0_bn1 -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>layer1_block1_add&#45;&gt;layer2_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1404.12,-78C1413.34,-78 1423.89,-78 1433.8,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1433.82,-81.5001 1443.82,-78 1433.82,-74.5001 1433.82,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1 -->\n",
       "<g id=\"node17\" class=\"node\"><title>layer2_block0_relu1</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1576,-96C1576,-96 1546,-96 1546,-96 1540,-96 1534,-90 1534,-84 1534,-84 1534,-72 1534,-72 1534,-66 1540,-60 1546,-60 1546,-60 1576,-60 1576,-60 1582,-60 1588,-66 1588,-72 1588,-72 1588,-84 1588,-84 1588,-90 1582,-96 1576,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1561\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1&#45;&gt;layer2_block0_relu1 -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>layer2_block0_bn1&#45;&gt;layer2_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1498.4,-78C1506.39,-78 1515.31,-78 1523.82,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1523.92,-81.5001 1533.92,-78 1523.92,-74.5001 1523.92,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1 -->\n",
       "<g id=\"node18\" class=\"node\"><title>layer2_block0_branch_conv1</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1666,-77C1666,-77 1636,-77 1636,-77 1630,-77 1624,-71 1624,-65 1624,-65 1624,-53 1624,-53 1624,-47 1630,-41 1636,-41 1636,-41 1666,-41 1666,-41 1672,-41 1678,-47 1678,-53 1678,-53 1678,-65 1678,-65 1678,-71 1672,-77 1666,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1651\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1 -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1588.4,-72.2994C1596.48,-70.5552 1605.51,-68.6065 1614.1,-66.7501\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1614.88,-70.1628 1623.92,-64.6311 1613.41,-63.3205 1614.88,-70.1628\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3 -->\n",
       "<g id=\"node22\" class=\"node\"><title>layer2_block0_conv3</title>\n",
       "<g id=\"a_node22\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-152C1936,-152 1906,-152 1906,-152 1900,-152 1894,-146 1894,-140 1894,-140 1894,-128 1894,-128 1894,-122 1900,-116 1906,-116 1906,-116 1936,-116 1936,-116 1942,-116 1948,-122 1948,-128 1948,-128 1948,-140 1948,-140 1948,-146 1942,-152 1936,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-130.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_conv3 -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1585.19,-96.1046C1594.3,-102.243 1605.16,-108.442 1616,-112 1708.77,-142.443 1825.74,-140.431 1883.83,-136.933\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1884.22,-140.415 1893.97,-136.27 1883.76,-133.43 1884.22,-140.415\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2 -->\n",
       "<g id=\"node19\" class=\"node\"><title>layer2_block0_branch_bn2</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1756,-77C1756,-77 1726,-77 1726,-77 1720,-77 1714,-71 1714,-65 1714,-65 1714,-53 1714,-53 1714,-47 1720,-41 1726,-41 1726,-41 1756,-41 1756,-41 1762,-41 1768,-47 1768,-53 1768,-53 1768,-65 1768,-65 1768,-71 1762,-77 1756,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2 -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1678.4,-59C1686.39,-59 1695.31,-59 1703.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1703.92,-62.5001 1713.92,-59 1703.92,-55.5001 1703.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2 -->\n",
       "<g id=\"node20\" class=\"node\"><title>layer2_block0_branch_relu2</title>\n",
       "<g id=\"a_node20\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1846,-77C1846,-77 1816,-77 1816,-77 1810,-77 1804,-71 1804,-65 1804,-65 1804,-53 1804,-53 1804,-47 1810,-41 1816,-41 1816,-41 1846,-41 1846,-41 1852,-41 1858,-47 1858,-53 1858,-53 1858,-65 1858,-65 1858,-71 1852,-77 1846,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1831\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2 -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1768.4,-59C1776.39,-59 1785.31,-59 1793.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1793.92,-62.5001 1803.92,-59 1793.92,-55.5001 1793.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2 -->\n",
       "<g id=\"node21\" class=\"node\"><title>layer2_block0_branch_conv2</title>\n",
       "<g id=\"a_node21\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-77C1936,-77 1906,-77 1906,-77 1900,-77 1894,-71 1894,-65 1894,-65 1894,-53 1894,-53 1894,-47 1900,-41 1906,-41 1906,-41 1936,-41 1936,-41 1942,-41 1948,-47 1948,-53 1948,-53 1948,-65 1948,-65 1948,-71 1942,-77 1936,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2 -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1858.4,-59C1866.39,-59 1875.31,-59 1883.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1883.92,-62.5001 1893.92,-59 1883.92,-55.5001 1883.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_add -->\n",
       "<g id=\"node23\" class=\"node\"><title>layer2_block0_add</title>\n",
       "<g id=\"a_node23\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2026,-97C2026,-97 1996,-97 1996,-97 1990,-97 1984,-91 1984,-85 1984,-85 1984,-73 1984,-73 1984,-67 1990,-61 1996,-61 1996,-61 2026,-61 2026,-61 2032,-61 2038,-67 2038,-73 2038,-73 2038,-85 2038,-85 2038,-91 2032,-97 2026,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2011\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>layer2_block0_branch_conv2&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-65.0007C1956.48,-66.8366 1965.51,-68.8879 1974.1,-70.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1973.39,-74.2692 1983.92,-73.0725 1974.94,-67.4432 1973.39,-74.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>layer2_block0_conv3&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-117.498C1956.83,-112.23 1966.29,-106.317 1975.22,-100.735\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1977.29,-103.569 1983.92,-95.3006 1973.58,-97.6327 1977.29,-103.569\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1 -->\n",
       "<g id=\"node24\" class=\"node\"><title>layer2_block1_bn1</title>\n",
       "<g id=\"a_node24\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2116,-97C2116,-97 2086,-97 2086,-97 2080,-97 2074,-91 2074,-85 2074,-85 2074,-73 2074,-73 2074,-67 2080,-61 2086,-61 2086,-61 2116,-61 2116,-61 2122,-61 2128,-67 2128,-73 2128,-73 2128,-85 2128,-85 2128,-91 2122,-97 2116,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2101\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_add&#45;&gt;layer2_block1_bn1 -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>layer2_block0_add&#45;&gt;layer2_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2038.4,-79C2046.39,-79 2055.31,-79 2063.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2063.92,-82.5001 2073.92,-79 2063.92,-75.5001 2063.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1 -->\n",
       "<g id=\"node25\" class=\"node\"><title>layer2_block1_relu1</title>\n",
       "<g id=\"a_node25\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2206,-97C2206,-97 2176,-97 2176,-97 2170,-97 2164,-91 2164,-85 2164,-85 2164,-73 2164,-73 2164,-67 2170,-61 2176,-61 2176,-61 2206,-61 2206,-61 2212,-61 2218,-67 2218,-73 2218,-73 2218,-85 2218,-85 2218,-91 2212,-97 2206,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2191\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1&#45;&gt;layer2_block1_relu1 -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>layer2_block1_bn1&#45;&gt;layer2_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2128.4,-79C2136.39,-79 2145.31,-79 2153.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2153.92,-82.5001 2163.92,-79 2153.92,-75.5001 2153.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1 -->\n",
       "<g id=\"node26\" class=\"node\"><title>layer2_block1_branch_conv1</title>\n",
       "<g id=\"a_node26\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2296,-82C2296,-82 2266,-82 2266,-82 2260,-82 2254,-76 2254,-70 2254,-70 2254,-58 2254,-58 2254,-52 2260,-46 2266,-46 2266,-46 2296,-46 2296,-46 2302,-46 2308,-52 2308,-58 2308,-58 2308,-70 2308,-70 2308,-76 2302,-82 2296,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2281\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1 -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2218.4,-74.4995C2226.39,-73.1375 2235.31,-71.6175 2243.82,-70.1664\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2244.65,-73.5763 2253.92,-68.4456 2243.47,-66.6758 2244.65,-73.5763\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_add -->\n",
       "<g id=\"node30\" class=\"node\"><title>layer2_block1_add</title>\n",
       "<g id=\"a_node30\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2656,-82C2656,-82 2626,-82 2626,-82 2620,-82 2614,-76 2614,-70 2614,-70 2614,-58 2614,-58 2614,-52 2620,-46 2626,-46 2626,-46 2656,-46 2656,-46 2662,-46 2668,-52 2668,-58 2668,-58 2668,-70 2668,-70 2668,-76 2662,-82 2656,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2641\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge33\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2211.74,-97.0337C2221.35,-104.671 2233.51,-112.769 2246,-117 2298.24,-134.703 2314.84,-122 2370,-122 2370,-122 2370,-122 2462,-122 2517.16,-122 2535.38,-138.893 2586,-117 2599.55,-111.141 2611.6,-100.216 2620.87,-89.7996\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2623.65,-91.9369 2627.39,-82.0259 2618.28,-87.4385 2623.65,-91.9369\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2 -->\n",
       "<g id=\"node27\" class=\"node\"><title>layer2_block1_branch_bn2</title>\n",
       "<g id=\"a_node27\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2386,-82C2386,-82 2356,-82 2356,-82 2350,-82 2344,-76 2344,-70 2344,-70 2344,-58 2344,-58 2344,-52 2350,-46 2356,-46 2356,-46 2386,-46 2386,-46 2392,-46 2398,-52 2398,-58 2398,-58 2398,-70 2398,-70 2398,-76 2392,-82 2386,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2 -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2308.4,-64C2316.39,-64 2325.31,-64 2333.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2333.92,-67.5001 2343.92,-64 2333.92,-60.5001 2333.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2 -->\n",
       "<g id=\"node28\" class=\"node\"><title>layer2_block1_branch_relu2</title>\n",
       "<g id=\"a_node28\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2476,-82C2476,-82 2446,-82 2446,-82 2440,-82 2434,-76 2434,-70 2434,-70 2434,-58 2434,-58 2434,-52 2440,-46 2446,-46 2446,-46 2476,-46 2476,-46 2482,-46 2488,-52 2488,-58 2488,-58 2488,-70 2488,-70 2488,-76 2482,-82 2476,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2461\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2 -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2398.4,-64C2406.39,-64 2415.31,-64 2423.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2423.92,-67.5001 2433.92,-64 2423.92,-60.5001 2423.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2 -->\n",
       "<g id=\"node29\" class=\"node\"><title>layer2_block1_branch_conv2</title>\n",
       "<g id=\"a_node29\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2566,-82C2566,-82 2536,-82 2536,-82 2530,-82 2524,-76 2524,-70 2524,-70 2524,-58 2524,-58 2524,-52 2530,-46 2536,-46 2536,-46 2566,-46 2566,-46 2572,-46 2578,-52 2578,-58 2578,-58 2578,-70 2578,-70 2578,-76 2572,-82 2566,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2551\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2 -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2488.4,-64C2496.39,-64 2505.31,-64 2513.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2513.92,-67.5001 2523.92,-64 2513.92,-60.5001 2513.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge34\" class=\"edge\"><title>layer2_block1_branch_conv2&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2578.4,-64C2586.39,-64 2595.31,-64 2603.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2603.92,-67.5001 2613.92,-64 2603.92,-60.5001 2603.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1 -->\n",
       "<g id=\"node31\" class=\"node\"><title>layer3_block0_bn1</title>\n",
       "<g id=\"a_node31\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2750,-82C2750,-82 2720,-82 2720,-82 2714,-82 2708,-76 2708,-70 2708,-70 2708,-58 2708,-58 2708,-52 2714,-46 2720,-46 2720,-46 2750,-46 2750,-46 2756,-46 2762,-52 2762,-58 2762,-58 2762,-70 2762,-70 2762,-76 2756,-82 2750,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2735\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_add&#45;&gt;layer3_block0_bn1 -->\n",
       "<g id=\"edge35\" class=\"edge\"><title>layer2_block1_add&#45;&gt;layer3_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2668.12,-64C2677.34,-64 2687.89,-64 2697.8,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2697.82,-67.5001 2707.82,-64 2697.82,-60.5001 2697.82,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1 -->\n",
       "<g id=\"node32\" class=\"node\"><title>layer3_block0_relu1</title>\n",
       "<g id=\"a_node32\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2840,-82C2840,-82 2810,-82 2810,-82 2804,-82 2798,-76 2798,-70 2798,-70 2798,-58 2798,-58 2798,-52 2804,-46 2810,-46 2810,-46 2840,-46 2840,-46 2846,-46 2852,-52 2852,-58 2852,-58 2852,-70 2852,-70 2852,-76 2846,-82 2840,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2825\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1&#45;&gt;layer3_block0_relu1 -->\n",
       "<g id=\"edge36\" class=\"edge\"><title>layer3_block0_bn1&#45;&gt;layer3_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2762.4,-64C2770.39,-64 2779.31,-64 2787.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2787.92,-67.5001 2797.92,-64 2787.92,-60.5001 2787.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1 -->\n",
       "<g id=\"node33\" class=\"node\"><title>layer3_block0_branch_conv1</title>\n",
       "<g id=\"a_node33\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2930,-72C2930,-72 2900,-72 2900,-72 2894,-72 2888,-66 2888,-60 2888,-60 2888,-48 2888,-48 2888,-42 2894,-36 2900,-36 2900,-36 2930,-36 2930,-36 2936,-36 2942,-42 2942,-48 2942,-48 2942,-60 2942,-60 2942,-66 2936,-72 2930,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"2915\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1 -->\n",
       "<g id=\"edge37\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2852.4,-60.9997C2860.39,-60.0917 2869.31,-59.0783 2877.82,-58.1109\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2878.38,-61.5706 2887.92,-56.9638 2877.59,-54.6153 2878.38,-61.5706\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3 -->\n",
       "<g id=\"node37\" class=\"node\"><title>layer3_block0_conv3</title>\n",
       "<g id=\"a_node37\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-147C3200,-147 3170,-147 3170,-147 3164,-147 3158,-141 3158,-135 3158,-135 3158,-123 3158,-123 3158,-117 3164,-111 3170,-111 3170,-111 3200,-111 3200,-111 3206,-111 3212,-117 3212,-123 3212,-123 3212,-135 3212,-135 3212,-141 3206,-147 3200,-147\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-125.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_conv3 -->\n",
       "<g id=\"edge41\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2843.22,-82.1953C2853.06,-91.336 2866.16,-101.594 2880,-107 2970.95,-142.522 3088.72,-138.267 3147.4,-133.121\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3148.02,-136.578 3157.65,-132.159 3147.37,-129.609 3148.02,-136.578\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2 -->\n",
       "<g id=\"node34\" class=\"node\"><title>layer3_block0_branch_bn2</title>\n",
       "<g id=\"a_node34\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3020,-72C3020,-72 2990,-72 2990,-72 2984,-72 2978,-66 2978,-60 2978,-60 2978,-48 2978,-48 2978,-42 2984,-36 2990,-36 2990,-36 3020,-36 3020,-36 3026,-36 3032,-42 3032,-48 3032,-48 3032,-60 3032,-60 3032,-66 3026,-72 3020,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2 -->\n",
       "<g id=\"edge38\" class=\"edge\"><title>layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2942.4,-54C2950.39,-54 2959.31,-54 2967.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2967.92,-57.5001 2977.92,-54 2967.92,-50.5001 2967.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2 -->\n",
       "<g id=\"node35\" class=\"node\"><title>layer3_block0_branch_relu2</title>\n",
       "<g id=\"a_node35\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3110,-72C3110,-72 3080,-72 3080,-72 3074,-72 3068,-66 3068,-60 3068,-60 3068,-48 3068,-48 3068,-42 3074,-36 3080,-36 3080,-36 3110,-36 3110,-36 3116,-36 3122,-42 3122,-48 3122,-48 3122,-60 3122,-60 3122,-66 3116,-72 3110,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3095\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2 -->\n",
       "<g id=\"edge39\" class=\"edge\"><title>layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3032.4,-54C3040.39,-54 3049.31,-54 3057.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3057.92,-57.5001 3067.92,-54 3057.92,-50.5001 3057.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2 -->\n",
       "<g id=\"node36\" class=\"node\"><title>layer3_block0_branch_conv2</title>\n",
       "<g id=\"a_node36\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-72C3200,-72 3170,-72 3170,-72 3164,-72 3158,-66 3158,-60 3158,-60 3158,-48 3158,-48 3158,-42 3164,-36 3170,-36 3170,-36 3200,-36 3200,-36 3206,-36 3212,-42 3212,-48 3212,-48 3212,-60 3212,-60 3212,-66 3206,-72 3200,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2 -->\n",
       "<g id=\"edge40\" class=\"edge\"><title>layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3122.4,-54C3130.39,-54 3139.31,-54 3147.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3147.92,-57.5001 3157.92,-54 3147.92,-50.5001 3147.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_add -->\n",
       "<g id=\"node38\" class=\"node\"><title>layer3_block0_add</title>\n",
       "<g id=\"a_node38\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3290,-109C3290,-109 3260,-109 3260,-109 3254,-109 3248,-103 3248,-97 3248,-97 3248,-85 3248,-85 3248,-79 3254,-73 3260,-73 3260,-73 3290,-73 3290,-73 3296,-73 3302,-79 3302,-85 3302,-85 3302,-97 3302,-97 3302,-103 3296,-109 3290,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3275\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge43\" class=\"edge\"><title>layer3_block0_branch_conv2&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-65.1012C3220.66,-68.5716 3229.9,-72.4578 3238.66,-76.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3237.34,-79.3846 3247.92,-80.0341 3240.06,-72.9318 3237.34,-79.3846\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge42\" class=\"edge\"><title>layer3_block0_conv3&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-117.599C3220.66,-114.035 3229.9,-110.043 3238.66,-106.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3240.13,-109.44 3247.92,-102.262 3237.35,-103.013 3240.13,-109.44\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1 -->\n",
       "<g id=\"node39\" class=\"node\"><title>layer3_block1_bn1</title>\n",
       "<g id=\"a_node39\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3380,-109C3380,-109 3350,-109 3350,-109 3344,-109 3338,-103 3338,-97 3338,-97 3338,-85 3338,-85 3338,-79 3344,-73 3350,-73 3350,-73 3380,-73 3380,-73 3386,-73 3392,-79 3392,-85 3392,-85 3392,-97 3392,-97 3392,-103 3386,-109 3380,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3365\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_add&#45;&gt;layer3_block1_bn1 -->\n",
       "<g id=\"edge44\" class=\"edge\"><title>layer3_block0_add&#45;&gt;layer3_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3302.4,-91C3310.39,-91 3319.31,-91 3327.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3327.92,-94.5001 3337.92,-91 3327.92,-87.5001 3327.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1 -->\n",
       "<g id=\"node40\" class=\"node\"><title>layer3_block1_relu1</title>\n",
       "<g id=\"a_node40\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3470,-109C3470,-109 3440,-109 3440,-109 3434,-109 3428,-103 3428,-97 3428,-97 3428,-85 3428,-85 3428,-79 3434,-73 3440,-73 3440,-73 3470,-73 3470,-73 3476,-73 3482,-79 3482,-85 3482,-85 3482,-97 3482,-97 3482,-103 3476,-109 3470,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3455\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1&#45;&gt;layer3_block1_relu1 -->\n",
       "<g id=\"edge45\" class=\"edge\"><title>layer3_block1_bn1&#45;&gt;layer3_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3392.4,-91C3400.39,-91 3409.31,-91 3417.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3417.92,-94.5001 3427.92,-91 3417.92,-87.5001 3417.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1 -->\n",
       "<g id=\"node41\" class=\"node\"><title>layer3_block1_branch_conv1</title>\n",
       "<g id=\"a_node41\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3560,-89C3560,-89 3530,-89 3530,-89 3524,-89 3518,-83 3518,-77 3518,-77 3518,-65 3518,-65 3518,-59 3524,-53 3530,-53 3530,-53 3560,-53 3560,-53 3566,-53 3572,-59 3572,-65 3572,-65 3572,-77 3572,-77 3572,-83 3566,-89 3560,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3545\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1 -->\n",
       "<g id=\"edge46\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3482.4,-84.9993C3490.48,-83.1634 3499.51,-81.1121 3508.1,-79.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3508.94,-82.5568 3517.92,-76.9275 3507.39,-75.7308 3508.94,-82.5568\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_add -->\n",
       "<g id=\"node45\" class=\"node\"><title>layer3_block1_add</title>\n",
       "<g id=\"a_node45\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3920,-109C3920,-109 3890,-109 3890,-109 3884,-109 3878,-103 3878,-97 3878,-97 3878,-85 3878,-85 3878,-79 3884,-73 3890,-73 3890,-73 3920,-73 3920,-73 3926,-73 3932,-79 3932,-85 3932,-85 3932,-97 3932,-97 3932,-103 3926,-109 3920,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3905\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge50\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3479.74,-109.102C3488.74,-114.964 3499.38,-120.78 3510,-124 3562.78,-140.006 3578.84,-129 3634,-129 3634,-129 3634,-129 3726,-129 3781.16,-129 3797.22,-140.006 3850,-124 3857.3,-121.786 3864.61,-118.345 3871.4,-114.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3873.55,-117.295 3880.26,-109.102 3869.9,-111.318 3873.55,-117.295\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2 -->\n",
       "<g id=\"node42\" class=\"node\"><title>layer3_block1_branch_bn2</title>\n",
       "<g id=\"a_node42\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3650,-89C3650,-89 3620,-89 3620,-89 3614,-89 3608,-83 3608,-77 3608,-77 3608,-65 3608,-65 3608,-59 3614,-53 3620,-53 3620,-53 3650,-53 3650,-53 3656,-53 3662,-59 3662,-65 3662,-65 3662,-77 3662,-77 3662,-83 3656,-89 3650,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2 -->\n",
       "<g id=\"edge47\" class=\"edge\"><title>layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3572.4,-71C3580.39,-71 3589.31,-71 3597.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3597.92,-74.5001 3607.92,-71 3597.92,-67.5001 3597.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2 -->\n",
       "<g id=\"node43\" class=\"node\"><title>layer3_block1_branch_relu2</title>\n",
       "<g id=\"a_node43\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3740,-89C3740,-89 3710,-89 3710,-89 3704,-89 3698,-83 3698,-77 3698,-77 3698,-65 3698,-65 3698,-59 3704,-53 3710,-53 3710,-53 3740,-53 3740,-53 3746,-53 3752,-59 3752,-65 3752,-65 3752,-77 3752,-77 3752,-83 3746,-89 3740,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3725\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2 -->\n",
       "<g id=\"edge48\" class=\"edge\"><title>layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3662.4,-71C3670.39,-71 3679.31,-71 3687.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3687.92,-74.5001 3697.92,-71 3687.92,-67.5001 3687.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2 -->\n",
       "<g id=\"node44\" class=\"node\"><title>layer3_block1_branch_conv2</title>\n",
       "<g id=\"a_node44\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3830,-89C3830,-89 3800,-89 3800,-89 3794,-89 3788,-83 3788,-77 3788,-77 3788,-65 3788,-65 3788,-59 3794,-53 3800,-53 3800,-53 3830,-53 3830,-53 3836,-53 3842,-59 3842,-65 3842,-65 3842,-77 3842,-77 3842,-83 3836,-89 3830,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3815\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2 -->\n",
       "<g id=\"edge49\" class=\"edge\"><title>layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3752.4,-71C3760.39,-71 3769.31,-71 3777.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3777.92,-74.5001 3787.92,-71 3777.92,-67.5001 3777.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge51\" class=\"edge\"><title>layer3_block1_branch_conv2&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3842.4,-77.0007C3850.48,-78.8366 3859.51,-80.8879 3868.1,-82.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3867.39,-86.2692 3877.92,-85.0725 3868.94,-79.4432 3867.39,-86.2692\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1 -->\n",
       "<g id=\"node46\" class=\"node\"><title>layer4_block0_bn1</title>\n",
       "<g id=\"a_node46\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4014,-109C4014,-109 3984,-109 3984,-109 3978,-109 3972,-103 3972,-97 3972,-97 3972,-85 3972,-85 3972,-79 3978,-73 3984,-73 3984,-73 4014,-73 4014,-73 4020,-73 4026,-79 4026,-85 4026,-85 4026,-97 4026,-97 4026,-103 4020,-109 4014,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3999\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_add&#45;&gt;layer4_block0_bn1 -->\n",
       "<g id=\"edge52\" class=\"edge\"><title>layer3_block1_add&#45;&gt;layer4_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3932.12,-91C3941.34,-91 3951.89,-91 3961.8,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3961.82,-94.5001 3971.82,-91 3961.82,-87.5001 3961.82,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1 -->\n",
       "<g id=\"node47\" class=\"node\"><title>layer4_block0_relu1</title>\n",
       "<g id=\"a_node47\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4104,-109C4104,-109 4074,-109 4074,-109 4068,-109 4062,-103 4062,-97 4062,-97 4062,-85 4062,-85 4062,-79 4068,-73 4074,-73 4074,-73 4104,-73 4104,-73 4110,-73 4116,-79 4116,-85 4116,-85 4116,-97 4116,-97 4116,-103 4110,-109 4104,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"4089\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1&#45;&gt;layer4_block0_relu1 -->\n",
       "<g id=\"edge53\" class=\"edge\"><title>layer4_block0_bn1&#45;&gt;layer4_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4026.4,-91C4034.39,-91 4043.31,-91 4051.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4051.92,-94.5001 4061.92,-91 4051.92,-87.5001 4051.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1 -->\n",
       "<g id=\"node48\" class=\"node\"><title>layer4_block0_branch_conv1</title>\n",
       "<g id=\"a_node48\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4194,-68C4194,-68 4164,-68 4164,-68 4158,-68 4152,-62 4152,-56 4152,-56 4152,-44 4152,-44 4152,-38 4158,-32 4164,-32 4164,-32 4194,-32 4194,-32 4200,-32 4206,-38 4206,-44 4206,-44 4206,-56 4206,-56 4206,-62 4200,-68 4194,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4179\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1 -->\n",
       "<g id=\"edge54\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.4,-78.6986C4124.66,-74.8531 4133.9,-70.5468 4142.66,-66.463\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4144.33,-69.5472 4151.92,-62.1514 4141.38,-63.2021 4144.33,-69.5472\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3 -->\n",
       "<g id=\"node52\" class=\"node\"><title>layer4_block0_conv3</title>\n",
       "<g id=\"a_node52\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-143C4464,-143 4434,-143 4434,-143 4428,-143 4422,-137 4422,-131 4422,-131 4422,-119 4422,-119 4422,-113 4428,-107 4434,-107 4434,-107 4464,-107 4464,-107 4470,-107 4476,-113 4476,-119 4476,-119 4476,-131 4476,-131 4476,-137 4470,-143 4464,-143\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-121.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_conv3 -->\n",
       "<g id=\"edge58\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.22,-97.6371C4125,-99.6421 4134.88,-101.664 4144,-103 4239.91,-117.051 4354.23,-122.258 4411.49,-124.083\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.73,-127.592 4421.83,-124.395 4411.94,-120.595 4411.73,-127.592\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2 -->\n",
       "<g id=\"node49\" class=\"node\"><title>layer4_block0_branch_bn2</title>\n",
       "<g id=\"a_node49\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4284,-68C4284,-68 4254,-68 4254,-68 4248,-68 4242,-62 4242,-56 4242,-56 4242,-44 4242,-44 4242,-38 4248,-32 4254,-32 4254,-32 4284,-32 4284,-32 4290,-32 4296,-38 4296,-44 4296,-44 4296,-56 4296,-56 4296,-62 4290,-68 4284,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2 -->\n",
       "<g id=\"edge55\" class=\"edge\"><title>layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4206.4,-50C4214.39,-50 4223.31,-50 4231.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4231.92,-53.5001 4241.92,-50 4231.92,-46.5001 4231.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2 -->\n",
       "<g id=\"node50\" class=\"node\"><title>layer4_block0_branch_relu2</title>\n",
       "<g id=\"a_node50\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4374,-68C4374,-68 4344,-68 4344,-68 4338,-68 4332,-62 4332,-56 4332,-56 4332,-44 4332,-44 4332,-38 4338,-32 4344,-32 4344,-32 4374,-32 4374,-32 4380,-32 4386,-38 4386,-44 4386,-44 4386,-56 4386,-56 4386,-62 4380,-68 4374,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4359\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2 -->\n",
       "<g id=\"edge56\" class=\"edge\"><title>layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4296.4,-50C4304.39,-50 4313.31,-50 4321.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4321.92,-53.5001 4331.92,-50 4321.92,-46.5001 4321.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2 -->\n",
       "<g id=\"node51\" class=\"node\"><title>layer4_block0_branch_conv2</title>\n",
       "<g id=\"a_node51\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-68C4464,-68 4434,-68 4434,-68 4428,-68 4422,-62 4422,-56 4422,-56 4422,-44 4422,-44 4422,-38 4428,-32 4434,-32 4434,-32 4464,-32 4464,-32 4470,-32 4476,-38 4476,-44 4476,-44 4476,-56 4476,-56 4476,-62 4470,-68 4464,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2 -->\n",
       "<g id=\"edge57\" class=\"edge\"><title>layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4386.4,-50C4394.39,-50 4403.31,-50 4411.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.92,-53.5001 4421.92,-50 4411.92,-46.5001 4411.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_add -->\n",
       "<g id=\"node53\" class=\"node\"><title>layer4_block0_add</title>\n",
       "<g id=\"a_node53\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M4554,-105C4554,-105 4524,-105 4524,-105 4518,-105 4512,-99 4512,-93 4512,-93 4512,-81 4512,-81 4512,-75 4518,-69 4524,-69 4524,-69 4554,-69 4554,-69 4560,-69 4566,-75 4566,-81 4566,-81 4566,-93 4566,-93 4566,-99 4560,-105 4554,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4539\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge60\" class=\"edge\"><title>layer4_block0_branch_conv2&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-61.1012C4484.66,-64.5716 4493.9,-68.4578 4502.66,-72.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4501.34,-75.3846 4511.92,-76.0341 4504.06,-68.9318 4501.34,-75.3846\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge59\" class=\"edge\"><title>layer4_block0_conv3&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-113.599C4484.66,-110.035 4493.9,-106.043 4502.66,-102.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4504.13,-105.44 4511.92,-98.2623 4501.35,-99.0135 4504.13,-105.44\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1 -->\n",
       "<g id=\"node54\" class=\"node\"><title>layer4_block1_bn1</title>\n",
       "<g id=\"a_node54\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4644,-105C4644,-105 4614,-105 4614,-105 4608,-105 4602,-99 4602,-93 4602,-93 4602,-81 4602,-81 4602,-75 4608,-69 4614,-69 4614,-69 4644,-69 4644,-69 4650,-69 4656,-75 4656,-81 4656,-81 4656,-93 4656,-93 4656,-99 4650,-105 4644,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4629\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_add&#45;&gt;layer4_block1_bn1 -->\n",
       "<g id=\"edge61\" class=\"edge\"><title>layer4_block0_add&#45;&gt;layer4_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4566.4,-87C4574.39,-87 4583.31,-87 4591.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4591.92,-90.5001 4601.92,-87 4591.92,-83.5001 4591.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1 -->\n",
       "<g id=\"node55\" class=\"node\"><title>layer4_block1_relu1</title>\n",
       "<g id=\"a_node55\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4734,-105C4734,-105 4704,-105 4704,-105 4698,-105 4692,-99 4692,-93 4692,-93 4692,-81 4692,-81 4692,-75 4698,-69 4704,-69 4704,-69 4734,-69 4734,-69 4740,-69 4746,-75 4746,-81 4746,-81 4746,-93 4746,-93 4746,-99 4740,-105 4734,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4719\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1&#45;&gt;layer4_block1_relu1 -->\n",
       "<g id=\"edge62\" class=\"edge\"><title>layer4_block1_bn1&#45;&gt;layer4_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4656.4,-87C4664.39,-87 4673.31,-87 4681.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4681.92,-90.5001 4691.92,-87 4681.92,-83.5001 4681.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1 -->\n",
       "<g id=\"node56\" class=\"node\"><title>layer4_block1_branch_conv1</title>\n",
       "<g id=\"a_node56\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4824,-85C4824,-85 4794,-85 4794,-85 4788,-85 4782,-79 4782,-73 4782,-73 4782,-61 4782,-61 4782,-55 4788,-49 4794,-49 4794,-49 4824,-49 4824,-49 4830,-49 4836,-55 4836,-61 4836,-61 4836,-73 4836,-73 4836,-79 4830,-85 4824,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4809\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1 -->\n",
       "<g id=\"edge63\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4746.4,-80.9993C4754.48,-79.1634 4763.51,-77.1121 4772.1,-75.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4772.94,-78.5568 4781.92,-72.9275 4771.39,-71.7308 4772.94,-78.5568\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_add -->\n",
       "<g id=\"node60\" class=\"node\"><title>layer4_block1_add</title>\n",
       "<g id=\"a_node60\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M5184,-105C5184,-105 5154,-105 5154,-105 5148,-105 5142,-99 5142,-93 5142,-93 5142,-81 5142,-81 5142,-75 5148,-69 5154,-69 5154,-69 5184,-69 5184,-69 5190,-69 5196,-75 5196,-81 5196,-81 5196,-93 5196,-93 5196,-99 5190,-105 5184,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5169\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge67\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4743.74,-105.102C4752.74,-110.964 4763.38,-116.78 4774,-120 4826.78,-136.006 4842.84,-125 4898,-125 4898,-125 4898,-125 4990,-125 5045.16,-125 5061.22,-136.006 5114,-120 5121.3,-117.786 5128.61,-114.345 5135.4,-110.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5137.55,-113.295 5144.26,-105.102 5133.9,-107.318 5137.55,-113.295\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2 -->\n",
       "<g id=\"node57\" class=\"node\"><title>layer4_block1_branch_bn2</title>\n",
       "<g id=\"a_node57\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4914,-85C4914,-85 4884,-85 4884,-85 4878,-85 4872,-79 4872,-73 4872,-73 4872,-61 4872,-61 4872,-55 4878,-49 4884,-49 4884,-49 4914,-49 4914,-49 4920,-49 4926,-55 4926,-61 4926,-61 4926,-73 4926,-73 4926,-79 4920,-85 4914,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2 -->\n",
       "<g id=\"edge64\" class=\"edge\"><title>layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4836.4,-67C4844.39,-67 4853.31,-67 4861.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4861.92,-70.5001 4871.92,-67 4861.92,-63.5001 4861.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2 -->\n",
       "<g id=\"node58\" class=\"node\"><title>layer4_block1_branch_relu2</title>\n",
       "<g id=\"a_node58\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M5004,-85C5004,-85 4974,-85 4974,-85 4968,-85 4962,-79 4962,-73 4962,-73 4962,-61 4962,-61 4962,-55 4968,-49 4974,-49 4974,-49 5004,-49 5004,-49 5010,-49 5016,-55 5016,-61 5016,-61 5016,-73 5016,-73 5016,-79 5010,-85 5004,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4989\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2 -->\n",
       "<g id=\"edge65\" class=\"edge\"><title>layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4926.4,-67C4934.39,-67 4943.31,-67 4951.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4951.92,-70.5001 4961.92,-67 4951.92,-63.5001 4951.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2 -->\n",
       "<g id=\"node59\" class=\"node\"><title>layer4_block1_branch_conv2</title>\n",
       "<g id=\"a_node59\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M5094,-85C5094,-85 5064,-85 5064,-85 5058,-85 5052,-79 5052,-73 5052,-73 5052,-61 5052,-61 5052,-55 5058,-49 5064,-49 5064,-49 5094,-49 5094,-49 5100,-49 5106,-55 5106,-61 5106,-61 5106,-73 5106,-73 5106,-79 5100,-85 5094,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"5079\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2 -->\n",
       "<g id=\"edge66\" class=\"edge\"><title>layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5016.4,-67C5024.39,-67 5033.31,-67 5041.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5041.92,-70.5001 5051.92,-67 5041.92,-63.5001 5041.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge68\" class=\"edge\"><title>layer4_block1_branch_conv2&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5106.4,-73.0007C5114.48,-74.8366 5123.51,-76.8879 5132.1,-78.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5131.39,-82.2692 5141.92,-81.0725 5132.94,-75.4432 5131.39,-82.2692\"/>\n",
       "</g>\n",
       "<!-- final_in -->\n",
       "<g id=\"node61\" class=\"node\"><title>final_in</title>\n",
       "<g id=\"a_node61\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5274,-105C5274,-105 5244,-105 5244,-105 5238,-105 5232,-99 5232,-93 5232,-93 5232,-81 5232,-81 5232,-75 5238,-69 5244,-69 5244,-69 5274,-69 5274,-69 5280,-69 5286,-75 5286,-81 5286,-81 5286,-93 5286,-93 5286,-99 5280,-105 5274,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5259\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">in</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_add&#45;&gt;final_in -->\n",
       "<g id=\"edge69\" class=\"edge\"><title>layer4_block1_add&#45;&gt;final_in</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5196.4,-87C5204.39,-87 5213.31,-87 5221.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5221.92,-90.5001 5231.92,-87 5221.92,-83.5001 5221.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_maxpool -->\n",
       "<g id=\"node62\" class=\"node\"><title>final_maxpool</title>\n",
       "<g id=\"a_node62\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#fdb462\" stroke=\"black\" d=\"M5375,-78C5375,-78 5334,-78 5334,-78 5328,-78 5322,-72 5322,-66 5322,-66 5322,-54 5322,-54 5322,-48 5328,-42 5334,-42 5334,-42 5375,-42 5375,-42 5381,-42 5387,-48 5387,-54 5387,-54 5387,-66 5387,-66 5387,-72 5381,-78 5375,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">maxpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_maxpool -->\n",
       "<g id=\"edge70\" class=\"edge\"><title>final_in&#45;&gt;final_maxpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-79.479C5294.13,-77.1436 5303.26,-74.509 5312.12,-71.9497\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.31,-75.2492 5321.94,-69.1121 5311.37,-68.524 5313.31,-75.2492\"/>\n",
       "</g>\n",
       "<!-- final_avgpool -->\n",
       "<g id=\"node63\" class=\"node\"><title>final_avgpool</title>\n",
       "<g id=\"a_node63\"><a xlink:title=\"AvgPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;ceil_mode&#39;: False, &#39;count_include_pad&#39;: True}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M5372.5,-132C5372.5,-132 5336.5,-132 5336.5,-132 5330.5,-132 5324.5,-126 5324.5,-120 5324.5,-120 5324.5,-108 5324.5,-108 5324.5,-102 5330.5,-96 5336.5,-96 5336.5,-96 5372.5,-96 5372.5,-96 5378.5,-96 5384.5,-102 5384.5,-108 5384.5,-108 5384.5,-120 5384.5,-120 5384.5,-126 5378.5,-132 5372.5,-132\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-110.3\" font-family=\"Times,serif\" font-size=\"14.00\">avgpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_avgpool -->\n",
       "<g id=\"edge71\" class=\"edge\"><title>final_in&#45;&gt;final_avgpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-94.521C5294.87,-97.0688 5304.92,-99.9726 5314.53,-102.746\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.73,-106.157 5324.3,-105.569 5315.67,-99.4322 5313.73,-106.157\"/>\n",
       "</g>\n",
       "<!-- final_concat -->\n",
       "<g id=\"node64\" class=\"node\"><title>final_concat</title>\n",
       "<g id=\"a_node64\"><a xlink:title=\"Concat {}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M5465,-105C5465,-105 5435,-105 5435,-105 5429,-105 5423,-99 5423,-93 5423,-93 5423,-81 5423,-81 5423,-75 5429,-69 5435,-69 5435,-69 5465,-69 5465,-69 5471,-69 5477,-75 5477,-81 5477,-81 5477,-93 5477,-93 5477,-99 5471,-105 5465,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5450\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">concat</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_maxpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge72\" class=\"edge\"><title>final_maxpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5387.15,-69.141C5395.53,-71.5601 5404.63,-74.1884 5413.21,-76.6659\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5412.39,-80.0717 5422.97,-79.4835 5414.33,-73.3465 5412.39,-80.0717\"/>\n",
       "</g>\n",
       "<!-- final_avgpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge73\" class=\"edge\"><title>final_avgpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5384.57,-105.605C5393.55,-103.013 5403.54,-100.127 5412.93,-97.4155\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5414.17,-100.702 5422.8,-94.5648 5412.22,-93.9766 5414.17,-100.702\"/>\n",
       "</g>\n",
       "<!-- final_flatten -->\n",
       "<g id=\"node65\" class=\"node\"><title>final_flatten</title>\n",
       "<g id=\"a_node65\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M5555,-105C5555,-105 5525,-105 5525,-105 5519,-105 5513,-99 5513,-93 5513,-93 5513,-81 5513,-81 5513,-75 5519,-69 5525,-69 5525,-69 5555,-69 5555,-69 5561,-69 5567,-75 5567,-81 5567,-81 5567,-93 5567,-93 5567,-99 5561,-105 5555,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5540\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_concat&#45;&gt;final_flatten -->\n",
       "<g id=\"edge74\" class=\"edge\"><title>final_concat&#45;&gt;final_flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5477.4,-87C5485.39,-87 5494.31,-87 5502.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5502.92,-90.5001 5512.92,-87 5502.92,-83.5001 5502.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_linear -->\n",
       "<g id=\"node66\" class=\"node\"><title>final_linear</title>\n",
       "<g id=\"a_node66\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: Parameter containing:\\ntensor([ 0.0293, &#45;0.0335, &#160;0.0371, &#45;0.0126, &#160;0.0422, &#45;0.0284, &#160;0.0414, &#45;0.0093,\\n &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#45;0.0033, &#45;0.0207], requires_grad=True)}\">\n",
       "<path fill=\"#ccebc5\" stroke=\"black\" d=\"M5645,-105C5645,-105 5615,-105 5615,-105 5609,-105 5603,-99 5603,-93 5603,-93 5603,-81 5603,-81 5603,-75 5609,-69 5615,-69 5615,-69 5645,-69 5645,-69 5651,-69 5657,-75 5657,-81 5657,-81 5657,-93 5657,-93 5657,-99 5651,-105 5645,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5630\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_flatten&#45;&gt;final_linear -->\n",
       "<g id=\"edge75\" class=\"edge\"><title>final_flatten&#45;&gt;final_linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5567.4,-87C5575.39,-87 5584.31,-87 5592.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5592.92,-90.5001 5602.92,-87 5592.92,-83.5001 5592.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node67\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node67\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5746,-105C5746,-105 5705,-105 5705,-105 5699,-105 5693,-99 5693,-93 5693,-93 5693,-81 5693,-81 5693,-75 5699,-69 5705,-69 5705,-69 5746,-69 5746,-69 5752,-69 5758,-75 5758,-81 5758,-81 5758,-93 5758,-93 5758,-99 5752,-105 5746,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5725.5\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_linear&#45;&gt;classifier -->\n",
       "<g id=\"edge76\" class=\"edge\"><title>final_linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5657.04,-87C5665.05,-87 5674.06,-87 5682.83,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5682.95,-90.5001 5692.94,-87 5682.94,-83.5001 5682.95,-90.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node68\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-96C42,-96 12,-96 12,-96 6,-96 0,-90 0,-84 0,-84 0,-72 0,-72 0,-66 6,-60 12,-60 12,-60 42,-60 42,-60 48,-60 54,-66 54,-72 54,-72 54,-84 54,-84 54,-90 48,-96 42,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-78C62.3932,-78 71.3106,-78 79.8241,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-81.5001 89.919,-78 79.919,-74.5001 79.919,-81.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7f0103d3c550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.0067       6.4659       1.6659       0.3804       0.3767       1.3605       0.5140       6.8426\n",
      "           2       0.0133       6.4757       1.0084       0.6408       0.3797       1.1952       0.6052      13.6980\n",
      "           3       0.0200       6.4757       0.7493       0.7383       0.3840       0.7589       0.7436      20.5577\n",
      "           4       0.0267       6.4743       0.6337       0.7812       0.3771       0.6412       0.7846      27.4091\n",
      "           5       0.0333       6.4726       0.5574       0.8073       0.3969       0.5612       0.8079      34.2787\n",
      "           6       0.0400       6.5163       0.5057       0.8265       0.4004       0.5388       0.8207      41.1955\n",
      "           7       0.0467       6.4917       0.4703       0.8377       0.3832       0.5971       0.7942      48.0704\n",
      "           8       0.0533       6.4733       0.4313       0.8520       0.3808       0.5451       0.8184      54.9244\n",
      "           9       0.0600       6.4730       0.4120       0.8571       0.3788       0.4883       0.8321      61.7763\n",
      "          10       0.0667       6.4733       0.3956       0.8628       0.3615       0.4864       0.8352      68.6111\n",
      "          11       0.0733       6.4686       0.3887       0.8677       0.3457       0.4207       0.8564      75.4254\n",
      "          12       0.0800       6.4676       0.3843       0.8682       0.3862       0.5289       0.8236      82.2791\n",
      "          13       0.0867       6.4701       0.3835       0.8679       0.3965       0.5583       0.8179      89.1457\n",
      "          14       0.0933       6.4668       0.3849       0.8660       0.3895       0.5378       0.8280      96.0020\n",
      "          15       0.1000       6.4638       0.3883       0.8676       0.3788       0.5946       0.8020     102.8446\n",
      "          16       0.0937       6.4681       0.3785       0.8692       0.3809       0.4908       0.8387     109.6937\n",
      "          17       0.0873       6.4647       0.3528       0.8778       0.3575       0.4324       0.8555     116.5159\n",
      "          18       0.0810       6.4637       0.3308       0.8866       0.3777       0.4448       0.8511     123.3573\n",
      "          19       0.0747       6.4648       0.3142       0.8918       0.3780       0.4033       0.8662     130.2001\n",
      "          20       0.0683       6.4620       0.2984       0.8982       0.3782       0.3999       0.8669     137.0403\n",
      "          21       0.0620       6.4629       0.2799       0.9043       0.3795       0.3456       0.8869     143.8828\n",
      "          22       0.0557       6.4639       0.2602       0.9083       0.3450       0.3891       0.8710     150.6918\n",
      "          23       0.0493       6.4608       0.2359       0.9181       0.3790       0.3440       0.8876     157.5316\n",
      "          24       0.0430       6.4667       0.2167       0.9245       0.3979       0.3152       0.8926     164.3962\n",
      "          25       0.0367       6.4809       0.1973       0.9323       0.3839       0.3002       0.8970     171.2610\n",
      "          26       0.0303       6.4654       0.1719       0.9410       0.3783       0.2919       0.9078     178.1046\n",
      "          27       0.0240       6.4614       0.1404       0.9502       0.3451       0.2545       0.9197     184.9111\n",
      "          28       0.0177       6.4605       0.1122       0.9605       0.3455       0.2335       0.9234     191.7170\n",
      "          29       0.0113       6.4619       0.0864       0.9704       0.3733       0.2394       0.9269     198.5522\n",
      "          30       0.0050       6.4648       0.0598       0.9798       0.3781       0.2031       0.9384     205.3952\n",
      "          31       0.0040       6.4613       0.0422       0.9870       0.3819       0.2010       0.9393     212.2385\n",
      "          32       0.0030       6.4626       0.0357       0.9892       0.3562       0.2060       0.9397     219.0572\n",
      "          33       0.0020       6.4622       0.0316       0.9904       0.3679       0.2008       0.9407     225.8874\n",
      "          34       0.0010       6.4617       0.0287       0.9916       0.3850       0.1993       0.9421     232.7341\n",
      "          35       0.0000       6.4642       0.0277       0.9924       0.3770       0.2009       0.9412     239.5752\n"
     ]
    }
   ],
   "source": [
    "lr_schedule = PiecewiseLinear([0, 15, 30, 35], [0, 0.1, 0.005, 0])\n",
    "batch_size = 128\n",
    "\n",
    "n = DAWN_net()\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device)\n",
    "#convert all children including batch norms to half precision (triggering slow codepath!)\n",
    "for v in model.children(): \n",
    "    v.half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR()])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 2: Mini-batches](https://www.myrtle.ai/2018/09/24/how_to_train_your_resnet_2/) - batch size=512 (256s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"43pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 43.08\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.187305 0.187305) rotate(0) translate(4 226)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-226 5762,-226 5762,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-52C94,-52 140,-52 140,-52 146,-52 152,-58 152,-64 152,-64 152,-115 152,-115 152,-121 146,-127 140,-127 140,-127 94,-127 94,-127 88,-127 82,-121 82,-115 82,-115 82,-64 82,-64 82,-58 88,-52 94,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M176,-16C176,-16 1408,-16 1408,-16 1414,-16 1420,-22 1420,-28 1420,-28 1420,-167 1420,-167 1420,-173 1414,-179 1408,-179 1408,-179 176,-179 176,-179 170,-179 164,-173 164,-167 164,-167 164,-28 164,-28 164,-22 170,-16 176,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"792\" y=\"-163.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer1_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M184,-24C184,-24 770,-24 770,-24 776,-24 782,-30 782,-36 782,-36 782,-136 782,-136 782,-142 776,-148 770,-148 770,-148 184,-148 184,-148 178,-148 172,-142 172,-136 172,-136 172,-36 172,-36 172,-30 178,-24 184,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer1_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-32C364,-32 680,-32 680,-32 686,-32 692,-38 692,-44 692,-44 692,-95 692,-95 692,-101 686,-107 680,-107 680,-107 364,-107 364,-107 358,-107 352,-101 352,-95 352,-95 352,-44 352,-44 352,-38 358,-32 364,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer1_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M814,-24C814,-24 1400,-24 1400,-24 1406,-24 1412,-30 1412,-36 1412,-36 1412,-136 1412,-136 1412,-142 1406,-148 1400,-148 1400,-148 814,-148 814,-148 808,-148 802,-142 802,-136 802,-136 802,-36 802,-36 802,-30 808,-24 814,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer1_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M994,-32C994,-32 1310,-32 1310,-32 1316,-32 1322,-38 1322,-44 1322,-44 1322,-95 1322,-95 1322,-101 1316,-107 1310,-107 1310,-107 994,-107 994,-107 988,-107 982,-101 982,-95 982,-95 982,-44 982,-44 982,-38 988,-32 994,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"1152\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1440,-17C1440,-17 2672,-17 2672,-17 2678,-17 2684,-23 2684,-29 2684,-29 2684,-202 2684,-202 2684,-208 2678,-214 2672,-214 2672,-214 1440,-214 1440,-214 1434,-214 1428,-208 1428,-202 1428,-202 1428,-29 1428,-29 1428,-23 1434,-17 1440,-17\"/>\n",
       "<text text-anchor=\"middle\" x=\"2056\" y=\"-198.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_layer2_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1448,-25C1448,-25 2034,-25 2034,-25 2040,-25 2046,-31 2046,-37 2046,-37 2046,-171 2046,-171 2046,-177 2040,-183 2034,-183 2034,-183 1448,-183 1448,-183 1442,-183 1436,-177 1436,-171 1436,-171 1436,-37 1436,-37 1436,-31 1442,-25 1448,-25\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-167.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_layer2_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1628,-33C1628,-33 1944,-33 1944,-33 1950,-33 1956,-39 1956,-45 1956,-45 1956,-96 1956,-96 1956,-102 1950,-108 1944,-108 1944,-108 1628,-108 1628,-108 1622,-108 1616,-102 1616,-96 1616,-96 1616,-45 1616,-45 1616,-39 1622,-33 1628,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"1786\" y=\"-92.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_layer2_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2078,-30C2078,-30 2664,-30 2664,-30 2670,-30 2676,-36 2676,-42 2676,-42 2676,-142 2676,-142 2676,-148 2670,-154 2664,-154 2664,-154 2078,-154 2078,-154 2072,-154 2066,-148 2066,-142 2066,-142 2066,-42 2066,-42 2066,-36 2072,-30 2078,-30\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-138.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust11\" class=\"cluster\"><title>cluster_layer2_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2258,-38C2258,-38 2574,-38 2574,-38 2580,-38 2586,-44 2586,-50 2586,-50 2586,-101 2586,-101 2586,-107 2580,-113 2574,-113 2574,-113 2258,-113 2258,-113 2252,-113 2246,-107 2246,-101 2246,-101 2246,-50 2246,-50 2246,-44 2252,-38 2258,-38\"/>\n",
       "<text text-anchor=\"middle\" x=\"2416\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust12\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2704,-12C2704,-12 3936,-12 3936,-12 3942,-12 3948,-18 3948,-24 3948,-24 3948,-197 3948,-197 3948,-203 3942,-209 3936,-209 3936,-209 2704,-209 2704,-209 2698,-209 2692,-203 2692,-197 2692,-197 2692,-24 2692,-24 2692,-18 2698,-12 2704,-12\"/>\n",
       "<text text-anchor=\"middle\" x=\"3320\" y=\"-193.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust13\" class=\"cluster\"><title>cluster_layer3_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2712,-20C2712,-20 3298,-20 3298,-20 3304,-20 3310,-26 3310,-32 3310,-32 3310,-166 3310,-166 3310,-172 3304,-178 3298,-178 3298,-178 2712,-178 2712,-178 2706,-178 2700,-172 2700,-166 2700,-166 2700,-32 2700,-32 2700,-26 2706,-20 2712,-20\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-162.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust14\" class=\"cluster\"><title>cluster_layer3_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2892,-28C2892,-28 3208,-28 3208,-28 3214,-28 3220,-34 3220,-40 3220,-40 3220,-91 3220,-91 3220,-97 3214,-103 3208,-103 3208,-103 2892,-103 2892,-103 2886,-103 2880,-97 2880,-91 2880,-91 2880,-40 2880,-40 2880,-34 2886,-28 2892,-28\"/>\n",
       "<text text-anchor=\"middle\" x=\"3050\" y=\"-87.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust15\" class=\"cluster\"><title>cluster_layer3_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3342,-37C3342,-37 3928,-37 3928,-37 3934,-37 3940,-43 3940,-49 3940,-49 3940,-149 3940,-149 3940,-155 3934,-161 3928,-161 3928,-161 3342,-161 3342,-161 3336,-161 3330,-155 3330,-149 3330,-149 3330,-49 3330,-49 3330,-43 3336,-37 3342,-37\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-145.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust16\" class=\"cluster\"><title>cluster_layer3_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3522,-45C3522,-45 3838,-45 3838,-45 3844,-45 3850,-51 3850,-57 3850,-57 3850,-108 3850,-108 3850,-114 3844,-120 3838,-120 3838,-120 3522,-120 3522,-120 3516,-120 3510,-114 3510,-108 3510,-108 3510,-57 3510,-57 3510,-51 3516,-45 3522,-45\"/>\n",
       "<text text-anchor=\"middle\" x=\"3680\" y=\"-104.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust17\" class=\"cluster\"><title>cluster_layer4</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3968,-8C3968,-8 5200,-8 5200,-8 5206,-8 5212,-14 5212,-20 5212,-20 5212,-193 5212,-193 5212,-199 5206,-205 5200,-205 5200,-205 3968,-205 3968,-205 3962,-205 3956,-199 3956,-193 3956,-193 3956,-20 3956,-20 3956,-14 3962,-8 3968,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"4584\" y=\"-189.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer4</text>\n",
       "</g>\n",
       "<g id=\"clust18\" class=\"cluster\"><title>cluster_layer4_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3976,-16C3976,-16 4562,-16 4562,-16 4568,-16 4574,-22 4574,-28 4574,-28 4574,-162 4574,-162 4574,-168 4568,-174 4562,-174 4562,-174 3976,-174 3976,-174 3970,-174 3964,-168 3964,-162 3964,-162 3964,-28 3964,-28 3964,-22 3970,-16 3976,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-158.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust19\" class=\"cluster\"><title>cluster_layer4_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4156,-24C4156,-24 4472,-24 4472,-24 4478,-24 4484,-30 4484,-36 4484,-36 4484,-87 4484,-87 4484,-93 4478,-99 4472,-99 4472,-99 4156,-99 4156,-99 4150,-99 4144,-93 4144,-87 4144,-87 4144,-36 4144,-36 4144,-30 4150,-24 4156,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"4314\" y=\"-83.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust20\" class=\"cluster\"><title>cluster_layer4_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4606,-33C4606,-33 5192,-33 5192,-33 5198,-33 5204,-39 5204,-45 5204,-45 5204,-145 5204,-145 5204,-151 5198,-157 5192,-157 5192,-157 4606,-157 4606,-157 4600,-157 4594,-151 4594,-145 4594,-145 4594,-45 4594,-45 4594,-39 4600,-33 4606,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-141.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust21\" class=\"cluster\"><title>cluster_layer4_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4786,-41C4786,-41 5102,-41 5102,-41 5108,-41 5114,-47 5114,-53 5114,-53 5114,-104 5114,-104 5114,-110 5108,-116 5102,-116 5102,-116 4786,-116 4786,-116 4780,-116 4774,-110 4774,-104 4774,-104 4774,-53 4774,-53 4774,-47 4780,-41 4786,-41\"/>\n",
       "<text text-anchor=\"middle\" x=\"4944\" y=\"-100.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust22\" class=\"cluster\"><title>cluster_final</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M5236,-34C5236,-34 5653,-34 5653,-34 5659,-34 5665,-40 5665,-46 5665,-46 5665,-151 5665,-151 5665,-157 5659,-163 5653,-163 5653,-163 5236,-163 5236,-163 5230,-163 5224,-157 5224,-151 5224,-151 5224,-46 5224,-46 5224,-40 5230,-34 5236,-34\"/>\n",
       "<text text-anchor=\"middle\" x=\"5444.5\" y=\"-147.8\" font-family=\"Times,serif\" font-size=\"14.00\">final</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-96C132,-96 102,-96 102,-96 96,-96 90,-90 90,-84 90,-84 90,-72 90,-72 90,-66 96,-60 102,-60 102,-60 132,-60 132,-60 138,-60 144,-66 144,-72 144,-72 144,-84 144,-84 144,-90 138,-96 132,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1 -->\n",
       "<g id=\"node2\" class=\"node\"><title>layer1_block0_bn1</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-96C222,-96 192,-96 192,-96 186,-96 180,-90 180,-84 180,-84 180,-72 180,-72 180,-66 186,-60 192,-60 192,-60 222,-60 222,-60 228,-60 234,-66 234,-72 234,-72 234,-84 234,-84 234,-90 228,-96 222,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;layer1_block0_bn1 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;layer1_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-78C152.393,-78 161.311,-78 169.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-81.5001 179.919,-78 169.919,-74.5001 169.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>layer1_block0_relu1</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-96C312,-96 282,-96 282,-96 276,-96 270,-90 270,-84 270,-84 270,-72 270,-72 270,-66 276,-60 282,-60 282,-60 312,-60 312,-60 318,-60 324,-66 324,-72 324,-72 324,-84 324,-84 324,-90 318,-96 312,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1&#45;&gt;layer1_block0_relu1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>layer1_block0_bn1&#45;&gt;layer1_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-78C242.393,-78 251.311,-78 259.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-81.5001 269.919,-78 259.919,-74.5001 259.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_block0_branch_conv1</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-76C402,-76 372,-76 372,-76 366,-76 360,-70 360,-64 360,-64 360,-52 360,-52 360,-46 366,-40 372,-40 372,-40 402,-40 402,-40 408,-40 414,-46 414,-52 414,-52 414,-64 414,-64 414,-70 408,-76 402,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-71.9993C332.481,-70.1634 341.507,-68.1121 350.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"350.943,-69.5568 359.919,-63.9275 349.392,-62.7308 350.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_add -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer1_block0_add</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M762,-96C762,-96 732,-96 732,-96 726,-96 720,-90 720,-84 720,-84 720,-72 720,-72 720,-66 726,-60 732,-60 732,-60 762,-60 762,-60 768,-60 774,-66 774,-72 774,-72 774,-84 774,-84 774,-90 768,-96 762,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"747\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M321.737,-96.1018C330.735,-101.964 341.381,-107.78 352,-111 404.782,-127.006 420.844,-116 476,-116 476,-116 476,-116 568,-116 623.156,-116 639.218,-127.006 692,-111 699.301,-108.786 706.614,-105.345 713.396,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"715.545,-104.295 722.263,-96.1018 711.902,-98.3176 715.545,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2 -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_block0_branch_bn2</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-76C492,-76 462,-76 462,-76 456,-76 450,-70 450,-64 450,-64 450,-52 450,-52 450,-46 456,-40 462,-40 462,-40 492,-40 492,-40 498,-40 504,-46 504,-52 504,-52 504,-64 504,-64 504,-70 498,-76 492,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-58C422.393,-58 431.311,-58 439.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-61.5001 449.919,-58 439.919,-54.5001 439.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2 -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer1_block0_branch_relu2</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-76C582,-76 552,-76 552,-76 546,-76 540,-70 540,-64 540,-64 540,-52 540,-52 540,-46 546,-40 552,-40 552,-40 582,-40 582,-40 588,-40 594,-46 594,-52 594,-52 594,-64 594,-64 594,-70 588,-76 582,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-58C512.393,-58 521.311,-58 529.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-61.5001 539.919,-58 529.919,-54.5001 529.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2 -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer1_block0_branch_conv2</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M672,-76C672,-76 642,-76 642,-76 636,-76 630,-70 630,-64 630,-64 630,-52 630,-52 630,-46 636,-40 642,-40 642,-40 672,-40 672,-40 678,-40 684,-46 684,-52 684,-52 684,-64 684,-64 684,-70 678,-76 672,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-58C602.393,-58 611.311,-58 619.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-61.5001 629.919,-58 619.919,-54.5001 619.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer1_block0_branch_conv2&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.403,-64.0007C692.481,-65.8366 701.507,-67.8879 710.105,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"709.392,-73.2692 719.919,-72.0725 710.943,-66.4432 709.392,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1 -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer1_block1_bn1</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M852,-96C852,-96 822,-96 822,-96 816,-96 810,-90 810,-84 810,-84 810,-72 810,-72 810,-66 816,-60 822,-60 822,-60 852,-60 852,-60 858,-60 864,-66 864,-72 864,-72 864,-84 864,-84 864,-90 858,-96 852,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_add&#45;&gt;layer1_block1_bn1 -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer1_block0_add&#45;&gt;layer1_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.403,-78C782.393,-78 791.311,-78 799.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"799.919,-81.5001 809.919,-78 799.919,-74.5001 799.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1 -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer1_block1_relu1</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M942,-96C942,-96 912,-96 912,-96 906,-96 900,-90 900,-84 900,-84 900,-72 900,-72 900,-66 906,-60 912,-60 912,-60 942,-60 942,-60 948,-60 954,-66 954,-72 954,-72 954,-84 954,-84 954,-90 948,-96 942,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1&#45;&gt;layer1_block1_relu1 -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer1_block1_bn1&#45;&gt;layer1_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.403,-78C872.393,-78 881.311,-78 889.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"889.919,-81.5001 899.919,-78 889.919,-74.5001 889.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1 -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer1_block1_branch_conv1</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1032,-76C1032,-76 1002,-76 1002,-76 996,-76 990,-70 990,-64 990,-64 990,-52 990,-52 990,-46 996,-40 1002,-40 1002,-40 1032,-40 1032,-40 1038,-40 1044,-46 1044,-52 1044,-52 1044,-64 1044,-64 1044,-70 1038,-76 1032,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1 -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M954.403,-71.9993C962.481,-70.1634 971.507,-68.1121 980.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"980.943,-69.5568 989.919,-63.9275 979.392,-62.7308 980.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_add -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer1_block1_add</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1392,-96C1392,-96 1362,-96 1362,-96 1356,-96 1350,-90 1350,-84 1350,-84 1350,-72 1350,-72 1350,-66 1356,-60 1362,-60 1362,-60 1392,-60 1392,-60 1398,-60 1404,-66 1404,-72 1404,-72 1404,-84 1404,-84 1404,-90 1398,-96 1392,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1377\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M951.737,-96.1018C960.735,-101.964 971.381,-107.78 982,-111 1034.78,-127.006 1050.84,-116 1106,-116 1106,-116 1106,-116 1198,-116 1253.16,-116 1269.22,-127.006 1322,-111 1329.3,-108.786 1336.61,-105.345 1343.4,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1345.55,-104.295 1352.26,-96.1018 1341.9,-98.3176 1345.55,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2 -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer1_block1_branch_bn2</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1122,-76C1122,-76 1092,-76 1092,-76 1086,-76 1080,-70 1080,-64 1080,-64 1080,-52 1080,-52 1080,-46 1086,-40 1092,-40 1092,-40 1122,-40 1122,-40 1128,-40 1134,-46 1134,-52 1134,-52 1134,-64 1134,-64 1134,-70 1128,-76 1122,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2 -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.4,-58C1052.39,-58 1061.31,-58 1069.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1069.92,-61.5001 1079.92,-58 1069.92,-54.5001 1069.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2 -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer1_block1_branch_relu2</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1212,-76C1212,-76 1182,-76 1182,-76 1176,-76 1170,-70 1170,-64 1170,-64 1170,-52 1170,-52 1170,-46 1176,-40 1182,-40 1182,-40 1212,-40 1212,-40 1218,-40 1224,-46 1224,-52 1224,-52 1224,-64 1224,-64 1224,-70 1218,-76 1212,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2 -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1134.4,-58C1142.39,-58 1151.31,-58 1159.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1159.92,-61.5001 1169.92,-58 1159.92,-54.5001 1159.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2 -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer1_block1_branch_conv2</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1302,-76C1302,-76 1272,-76 1272,-76 1266,-76 1260,-70 1260,-64 1260,-64 1260,-52 1260,-52 1260,-46 1266,-40 1272,-40 1272,-40 1302,-40 1302,-40 1308,-40 1314,-46 1314,-52 1314,-52 1314,-64 1314,-64 1314,-70 1308,-76 1302,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1287\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2 -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1224.4,-58C1232.39,-58 1241.31,-58 1249.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1249.92,-61.5001 1259.92,-58 1249.92,-54.5001 1249.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>layer1_block1_branch_conv2&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1314.4,-64.0007C1322.48,-65.8366 1331.51,-67.8879 1340.1,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1339.39,-73.2692 1349.92,-72.0725 1340.94,-66.4432 1339.39,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1 -->\n",
       "<g id=\"node16\" class=\"node\"><title>layer2_block0_bn1</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1486,-96C1486,-96 1456,-96 1456,-96 1450,-96 1444,-90 1444,-84 1444,-84 1444,-72 1444,-72 1444,-66 1450,-60 1456,-60 1456,-60 1486,-60 1486,-60 1492,-60 1498,-66 1498,-72 1498,-72 1498,-84 1498,-84 1498,-90 1492,-96 1486,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1471\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_add&#45;&gt;layer2_block0_bn1 -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>layer1_block1_add&#45;&gt;layer2_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1404.12,-78C1413.34,-78 1423.89,-78 1433.8,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1433.82,-81.5001 1443.82,-78 1433.82,-74.5001 1433.82,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1 -->\n",
       "<g id=\"node17\" class=\"node\"><title>layer2_block0_relu1</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1576,-96C1576,-96 1546,-96 1546,-96 1540,-96 1534,-90 1534,-84 1534,-84 1534,-72 1534,-72 1534,-66 1540,-60 1546,-60 1546,-60 1576,-60 1576,-60 1582,-60 1588,-66 1588,-72 1588,-72 1588,-84 1588,-84 1588,-90 1582,-96 1576,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1561\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1&#45;&gt;layer2_block0_relu1 -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>layer2_block0_bn1&#45;&gt;layer2_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1498.4,-78C1506.39,-78 1515.31,-78 1523.82,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1523.92,-81.5001 1533.92,-78 1523.92,-74.5001 1523.92,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1 -->\n",
       "<g id=\"node18\" class=\"node\"><title>layer2_block0_branch_conv1</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1666,-77C1666,-77 1636,-77 1636,-77 1630,-77 1624,-71 1624,-65 1624,-65 1624,-53 1624,-53 1624,-47 1630,-41 1636,-41 1636,-41 1666,-41 1666,-41 1672,-41 1678,-47 1678,-53 1678,-53 1678,-65 1678,-65 1678,-71 1672,-77 1666,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1651\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1 -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1588.4,-72.2994C1596.48,-70.5552 1605.51,-68.6065 1614.1,-66.7501\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1614.88,-70.1628 1623.92,-64.6311 1613.41,-63.3205 1614.88,-70.1628\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3 -->\n",
       "<g id=\"node22\" class=\"node\"><title>layer2_block0_conv3</title>\n",
       "<g id=\"a_node22\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-152C1936,-152 1906,-152 1906,-152 1900,-152 1894,-146 1894,-140 1894,-140 1894,-128 1894,-128 1894,-122 1900,-116 1906,-116 1906,-116 1936,-116 1936,-116 1942,-116 1948,-122 1948,-128 1948,-128 1948,-140 1948,-140 1948,-146 1942,-152 1936,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-130.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_conv3 -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1585.19,-96.1046C1594.3,-102.243 1605.16,-108.442 1616,-112 1708.77,-142.443 1825.74,-140.431 1883.83,-136.933\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1884.22,-140.415 1893.97,-136.27 1883.76,-133.43 1884.22,-140.415\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2 -->\n",
       "<g id=\"node19\" class=\"node\"><title>layer2_block0_branch_bn2</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1756,-77C1756,-77 1726,-77 1726,-77 1720,-77 1714,-71 1714,-65 1714,-65 1714,-53 1714,-53 1714,-47 1720,-41 1726,-41 1726,-41 1756,-41 1756,-41 1762,-41 1768,-47 1768,-53 1768,-53 1768,-65 1768,-65 1768,-71 1762,-77 1756,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2 -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1678.4,-59C1686.39,-59 1695.31,-59 1703.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1703.92,-62.5001 1713.92,-59 1703.92,-55.5001 1703.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2 -->\n",
       "<g id=\"node20\" class=\"node\"><title>layer2_block0_branch_relu2</title>\n",
       "<g id=\"a_node20\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1846,-77C1846,-77 1816,-77 1816,-77 1810,-77 1804,-71 1804,-65 1804,-65 1804,-53 1804,-53 1804,-47 1810,-41 1816,-41 1816,-41 1846,-41 1846,-41 1852,-41 1858,-47 1858,-53 1858,-53 1858,-65 1858,-65 1858,-71 1852,-77 1846,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1831\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2 -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1768.4,-59C1776.39,-59 1785.31,-59 1793.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1793.92,-62.5001 1803.92,-59 1793.92,-55.5001 1793.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2 -->\n",
       "<g id=\"node21\" class=\"node\"><title>layer2_block0_branch_conv2</title>\n",
       "<g id=\"a_node21\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-77C1936,-77 1906,-77 1906,-77 1900,-77 1894,-71 1894,-65 1894,-65 1894,-53 1894,-53 1894,-47 1900,-41 1906,-41 1906,-41 1936,-41 1936,-41 1942,-41 1948,-47 1948,-53 1948,-53 1948,-65 1948,-65 1948,-71 1942,-77 1936,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2 -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1858.4,-59C1866.39,-59 1875.31,-59 1883.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1883.92,-62.5001 1893.92,-59 1883.92,-55.5001 1883.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_add -->\n",
       "<g id=\"node23\" class=\"node\"><title>layer2_block0_add</title>\n",
       "<g id=\"a_node23\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2026,-97C2026,-97 1996,-97 1996,-97 1990,-97 1984,-91 1984,-85 1984,-85 1984,-73 1984,-73 1984,-67 1990,-61 1996,-61 1996,-61 2026,-61 2026,-61 2032,-61 2038,-67 2038,-73 2038,-73 2038,-85 2038,-85 2038,-91 2032,-97 2026,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2011\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>layer2_block0_branch_conv2&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-65.0007C1956.48,-66.8366 1965.51,-68.8879 1974.1,-70.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1973.39,-74.2692 1983.92,-73.0725 1974.94,-67.4432 1973.39,-74.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>layer2_block0_conv3&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-117.498C1956.83,-112.23 1966.29,-106.317 1975.22,-100.735\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1977.29,-103.569 1983.92,-95.3006 1973.58,-97.6327 1977.29,-103.569\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1 -->\n",
       "<g id=\"node24\" class=\"node\"><title>layer2_block1_bn1</title>\n",
       "<g id=\"a_node24\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2116,-97C2116,-97 2086,-97 2086,-97 2080,-97 2074,-91 2074,-85 2074,-85 2074,-73 2074,-73 2074,-67 2080,-61 2086,-61 2086,-61 2116,-61 2116,-61 2122,-61 2128,-67 2128,-73 2128,-73 2128,-85 2128,-85 2128,-91 2122,-97 2116,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2101\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_add&#45;&gt;layer2_block1_bn1 -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>layer2_block0_add&#45;&gt;layer2_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2038.4,-79C2046.39,-79 2055.31,-79 2063.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2063.92,-82.5001 2073.92,-79 2063.92,-75.5001 2063.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1 -->\n",
       "<g id=\"node25\" class=\"node\"><title>layer2_block1_relu1</title>\n",
       "<g id=\"a_node25\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2206,-97C2206,-97 2176,-97 2176,-97 2170,-97 2164,-91 2164,-85 2164,-85 2164,-73 2164,-73 2164,-67 2170,-61 2176,-61 2176,-61 2206,-61 2206,-61 2212,-61 2218,-67 2218,-73 2218,-73 2218,-85 2218,-85 2218,-91 2212,-97 2206,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2191\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1&#45;&gt;layer2_block1_relu1 -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>layer2_block1_bn1&#45;&gt;layer2_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2128.4,-79C2136.39,-79 2145.31,-79 2153.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2153.92,-82.5001 2163.92,-79 2153.92,-75.5001 2153.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1 -->\n",
       "<g id=\"node26\" class=\"node\"><title>layer2_block1_branch_conv1</title>\n",
       "<g id=\"a_node26\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2296,-82C2296,-82 2266,-82 2266,-82 2260,-82 2254,-76 2254,-70 2254,-70 2254,-58 2254,-58 2254,-52 2260,-46 2266,-46 2266,-46 2296,-46 2296,-46 2302,-46 2308,-52 2308,-58 2308,-58 2308,-70 2308,-70 2308,-76 2302,-82 2296,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2281\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1 -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2218.4,-74.4995C2226.39,-73.1375 2235.31,-71.6175 2243.82,-70.1664\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2244.65,-73.5763 2253.92,-68.4456 2243.47,-66.6758 2244.65,-73.5763\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_add -->\n",
       "<g id=\"node30\" class=\"node\"><title>layer2_block1_add</title>\n",
       "<g id=\"a_node30\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2656,-82C2656,-82 2626,-82 2626,-82 2620,-82 2614,-76 2614,-70 2614,-70 2614,-58 2614,-58 2614,-52 2620,-46 2626,-46 2626,-46 2656,-46 2656,-46 2662,-46 2668,-52 2668,-58 2668,-58 2668,-70 2668,-70 2668,-76 2662,-82 2656,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2641\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge33\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2211.74,-97.0337C2221.35,-104.671 2233.51,-112.769 2246,-117 2298.24,-134.703 2314.84,-122 2370,-122 2370,-122 2370,-122 2462,-122 2517.16,-122 2535.38,-138.893 2586,-117 2599.55,-111.141 2611.6,-100.216 2620.87,-89.7996\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2623.65,-91.9369 2627.39,-82.0259 2618.28,-87.4385 2623.65,-91.9369\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2 -->\n",
       "<g id=\"node27\" class=\"node\"><title>layer2_block1_branch_bn2</title>\n",
       "<g id=\"a_node27\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2386,-82C2386,-82 2356,-82 2356,-82 2350,-82 2344,-76 2344,-70 2344,-70 2344,-58 2344,-58 2344,-52 2350,-46 2356,-46 2356,-46 2386,-46 2386,-46 2392,-46 2398,-52 2398,-58 2398,-58 2398,-70 2398,-70 2398,-76 2392,-82 2386,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2 -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2308.4,-64C2316.39,-64 2325.31,-64 2333.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2333.92,-67.5001 2343.92,-64 2333.92,-60.5001 2333.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2 -->\n",
       "<g id=\"node28\" class=\"node\"><title>layer2_block1_branch_relu2</title>\n",
       "<g id=\"a_node28\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2476,-82C2476,-82 2446,-82 2446,-82 2440,-82 2434,-76 2434,-70 2434,-70 2434,-58 2434,-58 2434,-52 2440,-46 2446,-46 2446,-46 2476,-46 2476,-46 2482,-46 2488,-52 2488,-58 2488,-58 2488,-70 2488,-70 2488,-76 2482,-82 2476,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2461\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2 -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2398.4,-64C2406.39,-64 2415.31,-64 2423.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2423.92,-67.5001 2433.92,-64 2423.92,-60.5001 2423.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2 -->\n",
       "<g id=\"node29\" class=\"node\"><title>layer2_block1_branch_conv2</title>\n",
       "<g id=\"a_node29\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2566,-82C2566,-82 2536,-82 2536,-82 2530,-82 2524,-76 2524,-70 2524,-70 2524,-58 2524,-58 2524,-52 2530,-46 2536,-46 2536,-46 2566,-46 2566,-46 2572,-46 2578,-52 2578,-58 2578,-58 2578,-70 2578,-70 2578,-76 2572,-82 2566,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2551\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2 -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2488.4,-64C2496.39,-64 2505.31,-64 2513.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2513.92,-67.5001 2523.92,-64 2513.92,-60.5001 2513.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge34\" class=\"edge\"><title>layer2_block1_branch_conv2&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2578.4,-64C2586.39,-64 2595.31,-64 2603.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2603.92,-67.5001 2613.92,-64 2603.92,-60.5001 2603.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1 -->\n",
       "<g id=\"node31\" class=\"node\"><title>layer3_block0_bn1</title>\n",
       "<g id=\"a_node31\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2750,-82C2750,-82 2720,-82 2720,-82 2714,-82 2708,-76 2708,-70 2708,-70 2708,-58 2708,-58 2708,-52 2714,-46 2720,-46 2720,-46 2750,-46 2750,-46 2756,-46 2762,-52 2762,-58 2762,-58 2762,-70 2762,-70 2762,-76 2756,-82 2750,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2735\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_add&#45;&gt;layer3_block0_bn1 -->\n",
       "<g id=\"edge35\" class=\"edge\"><title>layer2_block1_add&#45;&gt;layer3_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2668.12,-64C2677.34,-64 2687.89,-64 2697.8,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2697.82,-67.5001 2707.82,-64 2697.82,-60.5001 2697.82,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1 -->\n",
       "<g id=\"node32\" class=\"node\"><title>layer3_block0_relu1</title>\n",
       "<g id=\"a_node32\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2840,-82C2840,-82 2810,-82 2810,-82 2804,-82 2798,-76 2798,-70 2798,-70 2798,-58 2798,-58 2798,-52 2804,-46 2810,-46 2810,-46 2840,-46 2840,-46 2846,-46 2852,-52 2852,-58 2852,-58 2852,-70 2852,-70 2852,-76 2846,-82 2840,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2825\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1&#45;&gt;layer3_block0_relu1 -->\n",
       "<g id=\"edge36\" class=\"edge\"><title>layer3_block0_bn1&#45;&gt;layer3_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2762.4,-64C2770.39,-64 2779.31,-64 2787.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2787.92,-67.5001 2797.92,-64 2787.92,-60.5001 2787.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1 -->\n",
       "<g id=\"node33\" class=\"node\"><title>layer3_block0_branch_conv1</title>\n",
       "<g id=\"a_node33\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2930,-72C2930,-72 2900,-72 2900,-72 2894,-72 2888,-66 2888,-60 2888,-60 2888,-48 2888,-48 2888,-42 2894,-36 2900,-36 2900,-36 2930,-36 2930,-36 2936,-36 2942,-42 2942,-48 2942,-48 2942,-60 2942,-60 2942,-66 2936,-72 2930,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"2915\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1 -->\n",
       "<g id=\"edge37\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2852.4,-60.9997C2860.39,-60.0917 2869.31,-59.0783 2877.82,-58.1109\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2878.38,-61.5706 2887.92,-56.9638 2877.59,-54.6153 2878.38,-61.5706\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3 -->\n",
       "<g id=\"node37\" class=\"node\"><title>layer3_block0_conv3</title>\n",
       "<g id=\"a_node37\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-147C3200,-147 3170,-147 3170,-147 3164,-147 3158,-141 3158,-135 3158,-135 3158,-123 3158,-123 3158,-117 3164,-111 3170,-111 3170,-111 3200,-111 3200,-111 3206,-111 3212,-117 3212,-123 3212,-123 3212,-135 3212,-135 3212,-141 3206,-147 3200,-147\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-125.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_conv3 -->\n",
       "<g id=\"edge41\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2843.22,-82.1953C2853.06,-91.336 2866.16,-101.594 2880,-107 2970.95,-142.522 3088.72,-138.267 3147.4,-133.121\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3148.02,-136.578 3157.65,-132.159 3147.37,-129.609 3148.02,-136.578\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2 -->\n",
       "<g id=\"node34\" class=\"node\"><title>layer3_block0_branch_bn2</title>\n",
       "<g id=\"a_node34\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3020,-72C3020,-72 2990,-72 2990,-72 2984,-72 2978,-66 2978,-60 2978,-60 2978,-48 2978,-48 2978,-42 2984,-36 2990,-36 2990,-36 3020,-36 3020,-36 3026,-36 3032,-42 3032,-48 3032,-48 3032,-60 3032,-60 3032,-66 3026,-72 3020,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2 -->\n",
       "<g id=\"edge38\" class=\"edge\"><title>layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2942.4,-54C2950.39,-54 2959.31,-54 2967.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2967.92,-57.5001 2977.92,-54 2967.92,-50.5001 2967.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2 -->\n",
       "<g id=\"node35\" class=\"node\"><title>layer3_block0_branch_relu2</title>\n",
       "<g id=\"a_node35\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3110,-72C3110,-72 3080,-72 3080,-72 3074,-72 3068,-66 3068,-60 3068,-60 3068,-48 3068,-48 3068,-42 3074,-36 3080,-36 3080,-36 3110,-36 3110,-36 3116,-36 3122,-42 3122,-48 3122,-48 3122,-60 3122,-60 3122,-66 3116,-72 3110,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3095\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2 -->\n",
       "<g id=\"edge39\" class=\"edge\"><title>layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3032.4,-54C3040.39,-54 3049.31,-54 3057.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3057.92,-57.5001 3067.92,-54 3057.92,-50.5001 3057.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2 -->\n",
       "<g id=\"node36\" class=\"node\"><title>layer3_block0_branch_conv2</title>\n",
       "<g id=\"a_node36\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-72C3200,-72 3170,-72 3170,-72 3164,-72 3158,-66 3158,-60 3158,-60 3158,-48 3158,-48 3158,-42 3164,-36 3170,-36 3170,-36 3200,-36 3200,-36 3206,-36 3212,-42 3212,-48 3212,-48 3212,-60 3212,-60 3212,-66 3206,-72 3200,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2 -->\n",
       "<g id=\"edge40\" class=\"edge\"><title>layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3122.4,-54C3130.39,-54 3139.31,-54 3147.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3147.92,-57.5001 3157.92,-54 3147.92,-50.5001 3147.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_add -->\n",
       "<g id=\"node38\" class=\"node\"><title>layer3_block0_add</title>\n",
       "<g id=\"a_node38\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3290,-109C3290,-109 3260,-109 3260,-109 3254,-109 3248,-103 3248,-97 3248,-97 3248,-85 3248,-85 3248,-79 3254,-73 3260,-73 3260,-73 3290,-73 3290,-73 3296,-73 3302,-79 3302,-85 3302,-85 3302,-97 3302,-97 3302,-103 3296,-109 3290,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3275\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge43\" class=\"edge\"><title>layer3_block0_branch_conv2&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-65.1012C3220.66,-68.5716 3229.9,-72.4578 3238.66,-76.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3237.34,-79.3846 3247.92,-80.0341 3240.06,-72.9318 3237.34,-79.3846\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge42\" class=\"edge\"><title>layer3_block0_conv3&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-117.599C3220.66,-114.035 3229.9,-110.043 3238.66,-106.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3240.13,-109.44 3247.92,-102.262 3237.35,-103.013 3240.13,-109.44\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1 -->\n",
       "<g id=\"node39\" class=\"node\"><title>layer3_block1_bn1</title>\n",
       "<g id=\"a_node39\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3380,-109C3380,-109 3350,-109 3350,-109 3344,-109 3338,-103 3338,-97 3338,-97 3338,-85 3338,-85 3338,-79 3344,-73 3350,-73 3350,-73 3380,-73 3380,-73 3386,-73 3392,-79 3392,-85 3392,-85 3392,-97 3392,-97 3392,-103 3386,-109 3380,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3365\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_add&#45;&gt;layer3_block1_bn1 -->\n",
       "<g id=\"edge44\" class=\"edge\"><title>layer3_block0_add&#45;&gt;layer3_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3302.4,-91C3310.39,-91 3319.31,-91 3327.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3327.92,-94.5001 3337.92,-91 3327.92,-87.5001 3327.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1 -->\n",
       "<g id=\"node40\" class=\"node\"><title>layer3_block1_relu1</title>\n",
       "<g id=\"a_node40\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3470,-109C3470,-109 3440,-109 3440,-109 3434,-109 3428,-103 3428,-97 3428,-97 3428,-85 3428,-85 3428,-79 3434,-73 3440,-73 3440,-73 3470,-73 3470,-73 3476,-73 3482,-79 3482,-85 3482,-85 3482,-97 3482,-97 3482,-103 3476,-109 3470,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3455\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1&#45;&gt;layer3_block1_relu1 -->\n",
       "<g id=\"edge45\" class=\"edge\"><title>layer3_block1_bn1&#45;&gt;layer3_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3392.4,-91C3400.39,-91 3409.31,-91 3417.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3417.92,-94.5001 3427.92,-91 3417.92,-87.5001 3417.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1 -->\n",
       "<g id=\"node41\" class=\"node\"><title>layer3_block1_branch_conv1</title>\n",
       "<g id=\"a_node41\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3560,-89C3560,-89 3530,-89 3530,-89 3524,-89 3518,-83 3518,-77 3518,-77 3518,-65 3518,-65 3518,-59 3524,-53 3530,-53 3530,-53 3560,-53 3560,-53 3566,-53 3572,-59 3572,-65 3572,-65 3572,-77 3572,-77 3572,-83 3566,-89 3560,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3545\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1 -->\n",
       "<g id=\"edge46\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3482.4,-84.9993C3490.48,-83.1634 3499.51,-81.1121 3508.1,-79.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3508.94,-82.5568 3517.92,-76.9275 3507.39,-75.7308 3508.94,-82.5568\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_add -->\n",
       "<g id=\"node45\" class=\"node\"><title>layer3_block1_add</title>\n",
       "<g id=\"a_node45\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3920,-109C3920,-109 3890,-109 3890,-109 3884,-109 3878,-103 3878,-97 3878,-97 3878,-85 3878,-85 3878,-79 3884,-73 3890,-73 3890,-73 3920,-73 3920,-73 3926,-73 3932,-79 3932,-85 3932,-85 3932,-97 3932,-97 3932,-103 3926,-109 3920,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3905\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge50\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3479.74,-109.102C3488.74,-114.964 3499.38,-120.78 3510,-124 3562.78,-140.006 3578.84,-129 3634,-129 3634,-129 3634,-129 3726,-129 3781.16,-129 3797.22,-140.006 3850,-124 3857.3,-121.786 3864.61,-118.345 3871.4,-114.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3873.55,-117.295 3880.26,-109.102 3869.9,-111.318 3873.55,-117.295\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2 -->\n",
       "<g id=\"node42\" class=\"node\"><title>layer3_block1_branch_bn2</title>\n",
       "<g id=\"a_node42\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3650,-89C3650,-89 3620,-89 3620,-89 3614,-89 3608,-83 3608,-77 3608,-77 3608,-65 3608,-65 3608,-59 3614,-53 3620,-53 3620,-53 3650,-53 3650,-53 3656,-53 3662,-59 3662,-65 3662,-65 3662,-77 3662,-77 3662,-83 3656,-89 3650,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2 -->\n",
       "<g id=\"edge47\" class=\"edge\"><title>layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3572.4,-71C3580.39,-71 3589.31,-71 3597.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3597.92,-74.5001 3607.92,-71 3597.92,-67.5001 3597.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2 -->\n",
       "<g id=\"node43\" class=\"node\"><title>layer3_block1_branch_relu2</title>\n",
       "<g id=\"a_node43\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3740,-89C3740,-89 3710,-89 3710,-89 3704,-89 3698,-83 3698,-77 3698,-77 3698,-65 3698,-65 3698,-59 3704,-53 3710,-53 3710,-53 3740,-53 3740,-53 3746,-53 3752,-59 3752,-65 3752,-65 3752,-77 3752,-77 3752,-83 3746,-89 3740,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3725\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2 -->\n",
       "<g id=\"edge48\" class=\"edge\"><title>layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3662.4,-71C3670.39,-71 3679.31,-71 3687.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3687.92,-74.5001 3697.92,-71 3687.92,-67.5001 3687.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2 -->\n",
       "<g id=\"node44\" class=\"node\"><title>layer3_block1_branch_conv2</title>\n",
       "<g id=\"a_node44\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3830,-89C3830,-89 3800,-89 3800,-89 3794,-89 3788,-83 3788,-77 3788,-77 3788,-65 3788,-65 3788,-59 3794,-53 3800,-53 3800,-53 3830,-53 3830,-53 3836,-53 3842,-59 3842,-65 3842,-65 3842,-77 3842,-77 3842,-83 3836,-89 3830,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3815\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2 -->\n",
       "<g id=\"edge49\" class=\"edge\"><title>layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3752.4,-71C3760.39,-71 3769.31,-71 3777.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3777.92,-74.5001 3787.92,-71 3777.92,-67.5001 3777.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge51\" class=\"edge\"><title>layer3_block1_branch_conv2&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3842.4,-77.0007C3850.48,-78.8366 3859.51,-80.8879 3868.1,-82.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3867.39,-86.2692 3877.92,-85.0725 3868.94,-79.4432 3867.39,-86.2692\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1 -->\n",
       "<g id=\"node46\" class=\"node\"><title>layer4_block0_bn1</title>\n",
       "<g id=\"a_node46\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4014,-109C4014,-109 3984,-109 3984,-109 3978,-109 3972,-103 3972,-97 3972,-97 3972,-85 3972,-85 3972,-79 3978,-73 3984,-73 3984,-73 4014,-73 4014,-73 4020,-73 4026,-79 4026,-85 4026,-85 4026,-97 4026,-97 4026,-103 4020,-109 4014,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3999\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_add&#45;&gt;layer4_block0_bn1 -->\n",
       "<g id=\"edge52\" class=\"edge\"><title>layer3_block1_add&#45;&gt;layer4_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3932.12,-91C3941.34,-91 3951.89,-91 3961.8,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3961.82,-94.5001 3971.82,-91 3961.82,-87.5001 3961.82,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1 -->\n",
       "<g id=\"node47\" class=\"node\"><title>layer4_block0_relu1</title>\n",
       "<g id=\"a_node47\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4104,-109C4104,-109 4074,-109 4074,-109 4068,-109 4062,-103 4062,-97 4062,-97 4062,-85 4062,-85 4062,-79 4068,-73 4074,-73 4074,-73 4104,-73 4104,-73 4110,-73 4116,-79 4116,-85 4116,-85 4116,-97 4116,-97 4116,-103 4110,-109 4104,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"4089\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1&#45;&gt;layer4_block0_relu1 -->\n",
       "<g id=\"edge53\" class=\"edge\"><title>layer4_block0_bn1&#45;&gt;layer4_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4026.4,-91C4034.39,-91 4043.31,-91 4051.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4051.92,-94.5001 4061.92,-91 4051.92,-87.5001 4051.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1 -->\n",
       "<g id=\"node48\" class=\"node\"><title>layer4_block0_branch_conv1</title>\n",
       "<g id=\"a_node48\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4194,-68C4194,-68 4164,-68 4164,-68 4158,-68 4152,-62 4152,-56 4152,-56 4152,-44 4152,-44 4152,-38 4158,-32 4164,-32 4164,-32 4194,-32 4194,-32 4200,-32 4206,-38 4206,-44 4206,-44 4206,-56 4206,-56 4206,-62 4200,-68 4194,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4179\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1 -->\n",
       "<g id=\"edge54\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.4,-78.6986C4124.66,-74.8531 4133.9,-70.5468 4142.66,-66.463\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4144.33,-69.5472 4151.92,-62.1514 4141.38,-63.2021 4144.33,-69.5472\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3 -->\n",
       "<g id=\"node52\" class=\"node\"><title>layer4_block0_conv3</title>\n",
       "<g id=\"a_node52\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-143C4464,-143 4434,-143 4434,-143 4428,-143 4422,-137 4422,-131 4422,-131 4422,-119 4422,-119 4422,-113 4428,-107 4434,-107 4434,-107 4464,-107 4464,-107 4470,-107 4476,-113 4476,-119 4476,-119 4476,-131 4476,-131 4476,-137 4470,-143 4464,-143\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-121.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_conv3 -->\n",
       "<g id=\"edge58\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.22,-97.6371C4125,-99.6421 4134.88,-101.664 4144,-103 4239.91,-117.051 4354.23,-122.258 4411.49,-124.083\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.73,-127.592 4421.83,-124.395 4411.94,-120.595 4411.73,-127.592\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2 -->\n",
       "<g id=\"node49\" class=\"node\"><title>layer4_block0_branch_bn2</title>\n",
       "<g id=\"a_node49\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4284,-68C4284,-68 4254,-68 4254,-68 4248,-68 4242,-62 4242,-56 4242,-56 4242,-44 4242,-44 4242,-38 4248,-32 4254,-32 4254,-32 4284,-32 4284,-32 4290,-32 4296,-38 4296,-44 4296,-44 4296,-56 4296,-56 4296,-62 4290,-68 4284,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2 -->\n",
       "<g id=\"edge55\" class=\"edge\"><title>layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4206.4,-50C4214.39,-50 4223.31,-50 4231.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4231.92,-53.5001 4241.92,-50 4231.92,-46.5001 4231.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2 -->\n",
       "<g id=\"node50\" class=\"node\"><title>layer4_block0_branch_relu2</title>\n",
       "<g id=\"a_node50\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4374,-68C4374,-68 4344,-68 4344,-68 4338,-68 4332,-62 4332,-56 4332,-56 4332,-44 4332,-44 4332,-38 4338,-32 4344,-32 4344,-32 4374,-32 4374,-32 4380,-32 4386,-38 4386,-44 4386,-44 4386,-56 4386,-56 4386,-62 4380,-68 4374,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4359\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2 -->\n",
       "<g id=\"edge56\" class=\"edge\"><title>layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4296.4,-50C4304.39,-50 4313.31,-50 4321.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4321.92,-53.5001 4331.92,-50 4321.92,-46.5001 4321.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2 -->\n",
       "<g id=\"node51\" class=\"node\"><title>layer4_block0_branch_conv2</title>\n",
       "<g id=\"a_node51\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-68C4464,-68 4434,-68 4434,-68 4428,-68 4422,-62 4422,-56 4422,-56 4422,-44 4422,-44 4422,-38 4428,-32 4434,-32 4434,-32 4464,-32 4464,-32 4470,-32 4476,-38 4476,-44 4476,-44 4476,-56 4476,-56 4476,-62 4470,-68 4464,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2 -->\n",
       "<g id=\"edge57\" class=\"edge\"><title>layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4386.4,-50C4394.39,-50 4403.31,-50 4411.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.92,-53.5001 4421.92,-50 4411.92,-46.5001 4411.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_add -->\n",
       "<g id=\"node53\" class=\"node\"><title>layer4_block0_add</title>\n",
       "<g id=\"a_node53\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M4554,-105C4554,-105 4524,-105 4524,-105 4518,-105 4512,-99 4512,-93 4512,-93 4512,-81 4512,-81 4512,-75 4518,-69 4524,-69 4524,-69 4554,-69 4554,-69 4560,-69 4566,-75 4566,-81 4566,-81 4566,-93 4566,-93 4566,-99 4560,-105 4554,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4539\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge60\" class=\"edge\"><title>layer4_block0_branch_conv2&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-61.1012C4484.66,-64.5716 4493.9,-68.4578 4502.66,-72.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4501.34,-75.3846 4511.92,-76.0341 4504.06,-68.9318 4501.34,-75.3846\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge59\" class=\"edge\"><title>layer4_block0_conv3&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-113.599C4484.66,-110.035 4493.9,-106.043 4502.66,-102.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4504.13,-105.44 4511.92,-98.2623 4501.35,-99.0135 4504.13,-105.44\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1 -->\n",
       "<g id=\"node54\" class=\"node\"><title>layer4_block1_bn1</title>\n",
       "<g id=\"a_node54\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4644,-105C4644,-105 4614,-105 4614,-105 4608,-105 4602,-99 4602,-93 4602,-93 4602,-81 4602,-81 4602,-75 4608,-69 4614,-69 4614,-69 4644,-69 4644,-69 4650,-69 4656,-75 4656,-81 4656,-81 4656,-93 4656,-93 4656,-99 4650,-105 4644,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4629\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_add&#45;&gt;layer4_block1_bn1 -->\n",
       "<g id=\"edge61\" class=\"edge\"><title>layer4_block0_add&#45;&gt;layer4_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4566.4,-87C4574.39,-87 4583.31,-87 4591.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4591.92,-90.5001 4601.92,-87 4591.92,-83.5001 4591.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1 -->\n",
       "<g id=\"node55\" class=\"node\"><title>layer4_block1_relu1</title>\n",
       "<g id=\"a_node55\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4734,-105C4734,-105 4704,-105 4704,-105 4698,-105 4692,-99 4692,-93 4692,-93 4692,-81 4692,-81 4692,-75 4698,-69 4704,-69 4704,-69 4734,-69 4734,-69 4740,-69 4746,-75 4746,-81 4746,-81 4746,-93 4746,-93 4746,-99 4740,-105 4734,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4719\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1&#45;&gt;layer4_block1_relu1 -->\n",
       "<g id=\"edge62\" class=\"edge\"><title>layer4_block1_bn1&#45;&gt;layer4_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4656.4,-87C4664.39,-87 4673.31,-87 4681.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4681.92,-90.5001 4691.92,-87 4681.92,-83.5001 4681.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1 -->\n",
       "<g id=\"node56\" class=\"node\"><title>layer4_block1_branch_conv1</title>\n",
       "<g id=\"a_node56\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4824,-85C4824,-85 4794,-85 4794,-85 4788,-85 4782,-79 4782,-73 4782,-73 4782,-61 4782,-61 4782,-55 4788,-49 4794,-49 4794,-49 4824,-49 4824,-49 4830,-49 4836,-55 4836,-61 4836,-61 4836,-73 4836,-73 4836,-79 4830,-85 4824,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4809\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1 -->\n",
       "<g id=\"edge63\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4746.4,-80.9993C4754.48,-79.1634 4763.51,-77.1121 4772.1,-75.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4772.94,-78.5568 4781.92,-72.9275 4771.39,-71.7308 4772.94,-78.5568\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_add -->\n",
       "<g id=\"node60\" class=\"node\"><title>layer4_block1_add</title>\n",
       "<g id=\"a_node60\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M5184,-105C5184,-105 5154,-105 5154,-105 5148,-105 5142,-99 5142,-93 5142,-93 5142,-81 5142,-81 5142,-75 5148,-69 5154,-69 5154,-69 5184,-69 5184,-69 5190,-69 5196,-75 5196,-81 5196,-81 5196,-93 5196,-93 5196,-99 5190,-105 5184,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5169\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge67\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4743.74,-105.102C4752.74,-110.964 4763.38,-116.78 4774,-120 4826.78,-136.006 4842.84,-125 4898,-125 4898,-125 4898,-125 4990,-125 5045.16,-125 5061.22,-136.006 5114,-120 5121.3,-117.786 5128.61,-114.345 5135.4,-110.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5137.55,-113.295 5144.26,-105.102 5133.9,-107.318 5137.55,-113.295\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2 -->\n",
       "<g id=\"node57\" class=\"node\"><title>layer4_block1_branch_bn2</title>\n",
       "<g id=\"a_node57\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4914,-85C4914,-85 4884,-85 4884,-85 4878,-85 4872,-79 4872,-73 4872,-73 4872,-61 4872,-61 4872,-55 4878,-49 4884,-49 4884,-49 4914,-49 4914,-49 4920,-49 4926,-55 4926,-61 4926,-61 4926,-73 4926,-73 4926,-79 4920,-85 4914,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2 -->\n",
       "<g id=\"edge64\" class=\"edge\"><title>layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4836.4,-67C4844.39,-67 4853.31,-67 4861.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4861.92,-70.5001 4871.92,-67 4861.92,-63.5001 4861.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2 -->\n",
       "<g id=\"node58\" class=\"node\"><title>layer4_block1_branch_relu2</title>\n",
       "<g id=\"a_node58\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M5004,-85C5004,-85 4974,-85 4974,-85 4968,-85 4962,-79 4962,-73 4962,-73 4962,-61 4962,-61 4962,-55 4968,-49 4974,-49 4974,-49 5004,-49 5004,-49 5010,-49 5016,-55 5016,-61 5016,-61 5016,-73 5016,-73 5016,-79 5010,-85 5004,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4989\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2 -->\n",
       "<g id=\"edge65\" class=\"edge\"><title>layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4926.4,-67C4934.39,-67 4943.31,-67 4951.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4951.92,-70.5001 4961.92,-67 4951.92,-63.5001 4951.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2 -->\n",
       "<g id=\"node59\" class=\"node\"><title>layer4_block1_branch_conv2</title>\n",
       "<g id=\"a_node59\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M5094,-85C5094,-85 5064,-85 5064,-85 5058,-85 5052,-79 5052,-73 5052,-73 5052,-61 5052,-61 5052,-55 5058,-49 5064,-49 5064,-49 5094,-49 5094,-49 5100,-49 5106,-55 5106,-61 5106,-61 5106,-73 5106,-73 5106,-79 5100,-85 5094,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"5079\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2 -->\n",
       "<g id=\"edge66\" class=\"edge\"><title>layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5016.4,-67C5024.39,-67 5033.31,-67 5041.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5041.92,-70.5001 5051.92,-67 5041.92,-63.5001 5041.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge68\" class=\"edge\"><title>layer4_block1_branch_conv2&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5106.4,-73.0007C5114.48,-74.8366 5123.51,-76.8879 5132.1,-78.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5131.39,-82.2692 5141.92,-81.0725 5132.94,-75.4432 5131.39,-82.2692\"/>\n",
       "</g>\n",
       "<!-- final_in -->\n",
       "<g id=\"node61\" class=\"node\"><title>final_in</title>\n",
       "<g id=\"a_node61\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5274,-105C5274,-105 5244,-105 5244,-105 5238,-105 5232,-99 5232,-93 5232,-93 5232,-81 5232,-81 5232,-75 5238,-69 5244,-69 5244,-69 5274,-69 5274,-69 5280,-69 5286,-75 5286,-81 5286,-81 5286,-93 5286,-93 5286,-99 5280,-105 5274,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5259\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">in</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_add&#45;&gt;final_in -->\n",
       "<g id=\"edge69\" class=\"edge\"><title>layer4_block1_add&#45;&gt;final_in</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5196.4,-87C5204.39,-87 5213.31,-87 5221.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5221.92,-90.5001 5231.92,-87 5221.92,-83.5001 5221.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_maxpool -->\n",
       "<g id=\"node62\" class=\"node\"><title>final_maxpool</title>\n",
       "<g id=\"a_node62\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#fdb462\" stroke=\"black\" d=\"M5375,-78C5375,-78 5334,-78 5334,-78 5328,-78 5322,-72 5322,-66 5322,-66 5322,-54 5322,-54 5322,-48 5328,-42 5334,-42 5334,-42 5375,-42 5375,-42 5381,-42 5387,-48 5387,-54 5387,-54 5387,-66 5387,-66 5387,-72 5381,-78 5375,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">maxpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_maxpool -->\n",
       "<g id=\"edge70\" class=\"edge\"><title>final_in&#45;&gt;final_maxpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-79.479C5294.13,-77.1436 5303.26,-74.509 5312.12,-71.9497\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.31,-75.2492 5321.94,-69.1121 5311.37,-68.524 5313.31,-75.2492\"/>\n",
       "</g>\n",
       "<!-- final_avgpool -->\n",
       "<g id=\"node63\" class=\"node\"><title>final_avgpool</title>\n",
       "<g id=\"a_node63\"><a xlink:title=\"AvgPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;ceil_mode&#39;: False, &#39;count_include_pad&#39;: True}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M5372.5,-132C5372.5,-132 5336.5,-132 5336.5,-132 5330.5,-132 5324.5,-126 5324.5,-120 5324.5,-120 5324.5,-108 5324.5,-108 5324.5,-102 5330.5,-96 5336.5,-96 5336.5,-96 5372.5,-96 5372.5,-96 5378.5,-96 5384.5,-102 5384.5,-108 5384.5,-108 5384.5,-120 5384.5,-120 5384.5,-126 5378.5,-132 5372.5,-132\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-110.3\" font-family=\"Times,serif\" font-size=\"14.00\">avgpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_avgpool -->\n",
       "<g id=\"edge71\" class=\"edge\"><title>final_in&#45;&gt;final_avgpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-94.521C5294.87,-97.0688 5304.92,-99.9726 5314.53,-102.746\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.73,-106.157 5324.3,-105.569 5315.67,-99.4322 5313.73,-106.157\"/>\n",
       "</g>\n",
       "<!-- final_concat -->\n",
       "<g id=\"node64\" class=\"node\"><title>final_concat</title>\n",
       "<g id=\"a_node64\"><a xlink:title=\"Concat {}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M5465,-105C5465,-105 5435,-105 5435,-105 5429,-105 5423,-99 5423,-93 5423,-93 5423,-81 5423,-81 5423,-75 5429,-69 5435,-69 5435,-69 5465,-69 5465,-69 5471,-69 5477,-75 5477,-81 5477,-81 5477,-93 5477,-93 5477,-99 5471,-105 5465,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5450\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">concat</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_maxpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge72\" class=\"edge\"><title>final_maxpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5387.15,-69.141C5395.53,-71.5601 5404.63,-74.1884 5413.21,-76.6659\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5412.39,-80.0717 5422.97,-79.4835 5414.33,-73.3465 5412.39,-80.0717\"/>\n",
       "</g>\n",
       "<!-- final_avgpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge73\" class=\"edge\"><title>final_avgpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5384.57,-105.605C5393.55,-103.013 5403.54,-100.127 5412.93,-97.4155\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5414.17,-100.702 5422.8,-94.5648 5412.22,-93.9766 5414.17,-100.702\"/>\n",
       "</g>\n",
       "<!-- final_flatten -->\n",
       "<g id=\"node65\" class=\"node\"><title>final_flatten</title>\n",
       "<g id=\"a_node65\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M5555,-105C5555,-105 5525,-105 5525,-105 5519,-105 5513,-99 5513,-93 5513,-93 5513,-81 5513,-81 5513,-75 5519,-69 5525,-69 5525,-69 5555,-69 5555,-69 5561,-69 5567,-75 5567,-81 5567,-81 5567,-93 5567,-93 5567,-99 5561,-105 5555,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5540\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_concat&#45;&gt;final_flatten -->\n",
       "<g id=\"edge74\" class=\"edge\"><title>final_concat&#45;&gt;final_flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5477.4,-87C5485.39,-87 5494.31,-87 5502.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5502.92,-90.5001 5512.92,-87 5502.92,-83.5001 5502.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_linear -->\n",
       "<g id=\"node66\" class=\"node\"><title>final_linear</title>\n",
       "<g id=\"a_node66\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: Parameter containing:\\ntensor([ 0.0075, &#160;0.0163, &#45;0.0053, &#45;0.0314, &#45;0.0397, &#45;0.0229, &#160;0.0104, &#160;0.0370,\\n &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#45;0.0271, &#160;0.0295], requires_grad=True)}\">\n",
       "<path fill=\"#ccebc5\" stroke=\"black\" d=\"M5645,-105C5645,-105 5615,-105 5615,-105 5609,-105 5603,-99 5603,-93 5603,-93 5603,-81 5603,-81 5603,-75 5609,-69 5615,-69 5615,-69 5645,-69 5645,-69 5651,-69 5657,-75 5657,-81 5657,-81 5657,-93 5657,-93 5657,-99 5651,-105 5645,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5630\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_flatten&#45;&gt;final_linear -->\n",
       "<g id=\"edge75\" class=\"edge\"><title>final_flatten&#45;&gt;final_linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5567.4,-87C5575.39,-87 5584.31,-87 5592.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5592.92,-90.5001 5602.92,-87 5592.92,-83.5001 5592.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node67\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node67\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5746,-105C5746,-105 5705,-105 5705,-105 5699,-105 5693,-99 5693,-93 5693,-93 5693,-81 5693,-81 5693,-75 5699,-69 5705,-69 5705,-69 5746,-69 5746,-69 5752,-69 5758,-75 5758,-81 5758,-81 5758,-93 5758,-93 5758,-99 5752,-105 5746,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5725.5\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_linear&#45;&gt;classifier -->\n",
       "<g id=\"edge76\" class=\"edge\"><title>final_linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5657.04,-87C5665.05,-87 5674.06,-87 5682.83,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5682.95,-90.5001 5692.94,-87 5682.94,-83.5001 5682.95,-90.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node68\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-96C42,-96 12,-96 12,-96 6,-96 0,-90 0,-84 0,-84 0,-72 0,-72 0,-66 6,-60 12,-60 12,-60 42,-60 42,-60 48,-60 54,-66 54,-72 54,-72 54,-84 54,-84 54,-90 48,-96 42,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-78C62.3932,-78 71.3106,-78 79.8241,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-81.5001 89.919,-78 79.919,-74.5001 79.919,-81.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7f0103de6f60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.0293       6.8122       1.7308       0.3570       0.4718       1.5902       0.4596       7.2841\n",
      "           2       0.0587       5.0096       1.0917       0.6081       0.3060       1.1059       0.6337      12.5997\n",
      "           3       0.0880       5.0059       0.7787       0.7243       0.3059       0.7754       0.7316      17.9115\n",
      "           4       0.1173       5.0505       0.6408       0.7758       0.3051       0.7841       0.7406      23.2671\n",
      "           5       0.1467       5.0028       0.5503       0.8097       0.3053       0.6362       0.7835      28.5752\n",
      "           6       0.1760       5.0016       0.4945       0.8294       0.3065       0.5512       0.8100      33.8833\n",
      "           7       0.2053       5.0035       0.4530       0.8440       0.3052       0.5276       0.8175      39.1920\n",
      "           8       0.2347       5.0277       0.4158       0.8562       0.3051       0.7159       0.7667      44.5249\n",
      "           9       0.2640       4.9965       0.4009       0.8623       0.3052       0.6988       0.7773      49.8265\n",
      "          10       0.2933       4.9949       0.3918       0.8649       0.3048       0.7618       0.7686      55.1263\n",
      "          11       0.3227       4.9955       0.3808       0.8688       0.3049       0.6022       0.8033      60.4267\n",
      "          12       0.3520       5.0080       0.3828       0.8687       0.3043       0.8023       0.7422      65.7390\n",
      "          13       0.3813       4.9926       0.3781       0.8692       0.3045       0.5326       0.8166      71.0361\n",
      "          14       0.4107       4.9907       0.3858       0.8684       0.3034       0.7230       0.7744      76.3303\n",
      "          15       0.4400       4.9852       0.3871       0.8676       0.3041       0.7347       0.7564      81.6195\n",
      "          16       0.4110       5.0139       0.3774       0.8693       0.3025       0.5324       0.8242      86.9359\n",
      "          17       0.3820       4.9816       0.3531       0.8781       0.3042       0.6754       0.7855      92.2217\n",
      "          18       0.3530       4.9814       0.3301       0.8853       0.3042       0.6634       0.7890      97.5072\n",
      "          19       0.3240       4.9827       0.3209       0.8892       0.3037       0.4133       0.8593     102.7936\n",
      "          20       0.2950       5.0155       0.2930       0.8992       0.3036       0.4125       0.8642     108.1126\n",
      "          21       0.2660       4.9856       0.2737       0.9041       0.3035       0.5012       0.8323     113.4017\n",
      "          22       0.2370       4.9837       0.2521       0.9133       0.3038       0.3844       0.8665     118.6892\n",
      "          23       0.2080       4.9812       0.2371       0.9180       0.3034       0.4030       0.8629     123.9738\n",
      "          24       0.1790       4.9962       0.2141       0.9256       0.3032       0.4333       0.8638     129.2732\n",
      "          25       0.1500       4.9815       0.1877       0.9347       0.3037       0.3146       0.8939     134.5585\n",
      "          26       0.1210       4.9809       0.1669       0.9426       0.3035       0.3234       0.8956     139.8429\n",
      "          27       0.0920       4.9824       0.1341       0.9546       0.3041       0.3062       0.9003     145.1293\n",
      "          28       0.0630       5.0103       0.1050       0.9634       0.3037       0.2467       0.9201     150.4434\n",
      "          29       0.0340       4.9807       0.0759       0.9740       0.3042       0.2205       0.9299     155.7283\n",
      "          30       0.0050       4.9807       0.0530       0.9824       0.3032       0.1940       0.9409     161.0122\n",
      "          31       0.0040       4.9801       0.0406       0.9880       0.3036       0.1920       0.9410     166.2959\n",
      "          32       0.0030       4.9928       0.0380       0.9880       0.3034       0.1933       0.9408     171.5920\n",
      "          33       0.0020       4.9809       0.0358       0.9894       0.3043       0.1934       0.9410     176.8772\n",
      "          34       0.0010       4.9810       0.0365       0.9890       0.3039       0.1925       0.9408     182.1621\n",
      "          35       0.0000       4.9800       0.0359       0.9894       0.3039       0.1931       0.9412     187.4460\n"
     ]
    }
   ],
   "source": [
    "lr_schedule = PiecewiseLinear([0, 15, 30, 35], [0, 0.44, 0.005, 0])\n",
    "batch_size = 512\n",
    "\n",
    "n = DAWN_net()\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device)\n",
    "#convert all children including batch norms to half precision (triggering slow codepath!)\n",
    "for v in model.children(): \n",
    "    v.half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR()])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 3: Regularisation](https://www.myrtle.ai/2018/09/24/how_to_train_your_resnet_3/) - speed up batch norms (186s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"43pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 43.08\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.187305 0.187305) rotate(0) translate(4 226)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-226 5762,-226 5762,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-52C94,-52 140,-52 140,-52 146,-52 152,-58 152,-64 152,-64 152,-115 152,-115 152,-121 146,-127 140,-127 140,-127 94,-127 94,-127 88,-127 82,-121 82,-115 82,-115 82,-64 82,-64 82,-58 88,-52 94,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M176,-16C176,-16 1408,-16 1408,-16 1414,-16 1420,-22 1420,-28 1420,-28 1420,-167 1420,-167 1420,-173 1414,-179 1408,-179 1408,-179 176,-179 176,-179 170,-179 164,-173 164,-167 164,-167 164,-28 164,-28 164,-22 170,-16 176,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"792\" y=\"-163.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer1_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M184,-24C184,-24 770,-24 770,-24 776,-24 782,-30 782,-36 782,-36 782,-136 782,-136 782,-142 776,-148 770,-148 770,-148 184,-148 184,-148 178,-148 172,-142 172,-136 172,-136 172,-36 172,-36 172,-30 178,-24 184,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer1_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-32C364,-32 680,-32 680,-32 686,-32 692,-38 692,-44 692,-44 692,-95 692,-95 692,-101 686,-107 680,-107 680,-107 364,-107 364,-107 358,-107 352,-101 352,-95 352,-95 352,-44 352,-44 352,-38 358,-32 364,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer1_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M814,-24C814,-24 1400,-24 1400,-24 1406,-24 1412,-30 1412,-36 1412,-36 1412,-136 1412,-136 1412,-142 1406,-148 1400,-148 1400,-148 814,-148 814,-148 808,-148 802,-142 802,-136 802,-136 802,-36 802,-36 802,-30 808,-24 814,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer1_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M994,-32C994,-32 1310,-32 1310,-32 1316,-32 1322,-38 1322,-44 1322,-44 1322,-95 1322,-95 1322,-101 1316,-107 1310,-107 1310,-107 994,-107 994,-107 988,-107 982,-101 982,-95 982,-95 982,-44 982,-44 982,-38 988,-32 994,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"1152\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1440,-17C1440,-17 2672,-17 2672,-17 2678,-17 2684,-23 2684,-29 2684,-29 2684,-202 2684,-202 2684,-208 2678,-214 2672,-214 2672,-214 1440,-214 1440,-214 1434,-214 1428,-208 1428,-202 1428,-202 1428,-29 1428,-29 1428,-23 1434,-17 1440,-17\"/>\n",
       "<text text-anchor=\"middle\" x=\"2056\" y=\"-198.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_layer2_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1448,-25C1448,-25 2034,-25 2034,-25 2040,-25 2046,-31 2046,-37 2046,-37 2046,-171 2046,-171 2046,-177 2040,-183 2034,-183 2034,-183 1448,-183 1448,-183 1442,-183 1436,-177 1436,-171 1436,-171 1436,-37 1436,-37 1436,-31 1442,-25 1448,-25\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-167.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_layer2_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1628,-33C1628,-33 1944,-33 1944,-33 1950,-33 1956,-39 1956,-45 1956,-45 1956,-96 1956,-96 1956,-102 1950,-108 1944,-108 1944,-108 1628,-108 1628,-108 1622,-108 1616,-102 1616,-96 1616,-96 1616,-45 1616,-45 1616,-39 1622,-33 1628,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"1786\" y=\"-92.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_layer2_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2078,-30C2078,-30 2664,-30 2664,-30 2670,-30 2676,-36 2676,-42 2676,-42 2676,-142 2676,-142 2676,-148 2670,-154 2664,-154 2664,-154 2078,-154 2078,-154 2072,-154 2066,-148 2066,-142 2066,-142 2066,-42 2066,-42 2066,-36 2072,-30 2078,-30\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-138.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust11\" class=\"cluster\"><title>cluster_layer2_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2258,-38C2258,-38 2574,-38 2574,-38 2580,-38 2586,-44 2586,-50 2586,-50 2586,-101 2586,-101 2586,-107 2580,-113 2574,-113 2574,-113 2258,-113 2258,-113 2252,-113 2246,-107 2246,-101 2246,-101 2246,-50 2246,-50 2246,-44 2252,-38 2258,-38\"/>\n",
       "<text text-anchor=\"middle\" x=\"2416\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust12\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2704,-12C2704,-12 3936,-12 3936,-12 3942,-12 3948,-18 3948,-24 3948,-24 3948,-197 3948,-197 3948,-203 3942,-209 3936,-209 3936,-209 2704,-209 2704,-209 2698,-209 2692,-203 2692,-197 2692,-197 2692,-24 2692,-24 2692,-18 2698,-12 2704,-12\"/>\n",
       "<text text-anchor=\"middle\" x=\"3320\" y=\"-193.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust13\" class=\"cluster\"><title>cluster_layer3_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2712,-20C2712,-20 3298,-20 3298,-20 3304,-20 3310,-26 3310,-32 3310,-32 3310,-166 3310,-166 3310,-172 3304,-178 3298,-178 3298,-178 2712,-178 2712,-178 2706,-178 2700,-172 2700,-166 2700,-166 2700,-32 2700,-32 2700,-26 2706,-20 2712,-20\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-162.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust14\" class=\"cluster\"><title>cluster_layer3_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2892,-28C2892,-28 3208,-28 3208,-28 3214,-28 3220,-34 3220,-40 3220,-40 3220,-91 3220,-91 3220,-97 3214,-103 3208,-103 3208,-103 2892,-103 2892,-103 2886,-103 2880,-97 2880,-91 2880,-91 2880,-40 2880,-40 2880,-34 2886,-28 2892,-28\"/>\n",
       "<text text-anchor=\"middle\" x=\"3050\" y=\"-87.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust15\" class=\"cluster\"><title>cluster_layer3_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3342,-37C3342,-37 3928,-37 3928,-37 3934,-37 3940,-43 3940,-49 3940,-49 3940,-149 3940,-149 3940,-155 3934,-161 3928,-161 3928,-161 3342,-161 3342,-161 3336,-161 3330,-155 3330,-149 3330,-149 3330,-49 3330,-49 3330,-43 3336,-37 3342,-37\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-145.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust16\" class=\"cluster\"><title>cluster_layer3_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3522,-45C3522,-45 3838,-45 3838,-45 3844,-45 3850,-51 3850,-57 3850,-57 3850,-108 3850,-108 3850,-114 3844,-120 3838,-120 3838,-120 3522,-120 3522,-120 3516,-120 3510,-114 3510,-108 3510,-108 3510,-57 3510,-57 3510,-51 3516,-45 3522,-45\"/>\n",
       "<text text-anchor=\"middle\" x=\"3680\" y=\"-104.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust17\" class=\"cluster\"><title>cluster_layer4</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3968,-8C3968,-8 5200,-8 5200,-8 5206,-8 5212,-14 5212,-20 5212,-20 5212,-193 5212,-193 5212,-199 5206,-205 5200,-205 5200,-205 3968,-205 3968,-205 3962,-205 3956,-199 3956,-193 3956,-193 3956,-20 3956,-20 3956,-14 3962,-8 3968,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"4584\" y=\"-189.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer4</text>\n",
       "</g>\n",
       "<g id=\"clust18\" class=\"cluster\"><title>cluster_layer4_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3976,-16C3976,-16 4562,-16 4562,-16 4568,-16 4574,-22 4574,-28 4574,-28 4574,-162 4574,-162 4574,-168 4568,-174 4562,-174 4562,-174 3976,-174 3976,-174 3970,-174 3964,-168 3964,-162 3964,-162 3964,-28 3964,-28 3964,-22 3970,-16 3976,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-158.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust19\" class=\"cluster\"><title>cluster_layer4_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4156,-24C4156,-24 4472,-24 4472,-24 4478,-24 4484,-30 4484,-36 4484,-36 4484,-87 4484,-87 4484,-93 4478,-99 4472,-99 4472,-99 4156,-99 4156,-99 4150,-99 4144,-93 4144,-87 4144,-87 4144,-36 4144,-36 4144,-30 4150,-24 4156,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"4314\" y=\"-83.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust20\" class=\"cluster\"><title>cluster_layer4_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4606,-33C4606,-33 5192,-33 5192,-33 5198,-33 5204,-39 5204,-45 5204,-45 5204,-145 5204,-145 5204,-151 5198,-157 5192,-157 5192,-157 4606,-157 4606,-157 4600,-157 4594,-151 4594,-145 4594,-145 4594,-45 4594,-45 4594,-39 4600,-33 4606,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-141.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust21\" class=\"cluster\"><title>cluster_layer4_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4786,-41C4786,-41 5102,-41 5102,-41 5108,-41 5114,-47 5114,-53 5114,-53 5114,-104 5114,-104 5114,-110 5108,-116 5102,-116 5102,-116 4786,-116 4786,-116 4780,-116 4774,-110 4774,-104 4774,-104 4774,-53 4774,-53 4774,-47 4780,-41 4786,-41\"/>\n",
       "<text text-anchor=\"middle\" x=\"4944\" y=\"-100.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust22\" class=\"cluster\"><title>cluster_final</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M5236,-34C5236,-34 5653,-34 5653,-34 5659,-34 5665,-40 5665,-46 5665,-46 5665,-151 5665,-151 5665,-157 5659,-163 5653,-163 5653,-163 5236,-163 5236,-163 5230,-163 5224,-157 5224,-151 5224,-151 5224,-46 5224,-46 5224,-40 5230,-34 5236,-34\"/>\n",
       "<text text-anchor=\"middle\" x=\"5444.5\" y=\"-147.8\" font-family=\"Times,serif\" font-size=\"14.00\">final</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-96C132,-96 102,-96 102,-96 96,-96 90,-90 90,-84 90,-84 90,-72 90,-72 90,-66 96,-60 102,-60 102,-60 132,-60 132,-60 138,-60 144,-66 144,-72 144,-72 144,-84 144,-84 144,-90 138,-96 132,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1 -->\n",
       "<g id=\"node2\" class=\"node\"><title>layer1_block0_bn1</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-96C222,-96 192,-96 192,-96 186,-96 180,-90 180,-84 180,-84 180,-72 180,-72 180,-66 186,-60 192,-60 192,-60 222,-60 222,-60 228,-60 234,-66 234,-72 234,-72 234,-84 234,-84 234,-90 228,-96 222,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;layer1_block0_bn1 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;layer1_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-78C152.393,-78 161.311,-78 169.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-81.5001 179.919,-78 169.919,-74.5001 169.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>layer1_block0_relu1</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-96C312,-96 282,-96 282,-96 276,-96 270,-90 270,-84 270,-84 270,-72 270,-72 270,-66 276,-60 282,-60 282,-60 312,-60 312,-60 318,-60 324,-66 324,-72 324,-72 324,-84 324,-84 324,-90 318,-96 312,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1&#45;&gt;layer1_block0_relu1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>layer1_block0_bn1&#45;&gt;layer1_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-78C242.393,-78 251.311,-78 259.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-81.5001 269.919,-78 259.919,-74.5001 259.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_block0_branch_conv1</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-76C402,-76 372,-76 372,-76 366,-76 360,-70 360,-64 360,-64 360,-52 360,-52 360,-46 366,-40 372,-40 372,-40 402,-40 402,-40 408,-40 414,-46 414,-52 414,-52 414,-64 414,-64 414,-70 408,-76 402,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-71.9993C332.481,-70.1634 341.507,-68.1121 350.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"350.943,-69.5568 359.919,-63.9275 349.392,-62.7308 350.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_add -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer1_block0_add</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M762,-96C762,-96 732,-96 732,-96 726,-96 720,-90 720,-84 720,-84 720,-72 720,-72 720,-66 726,-60 732,-60 732,-60 762,-60 762,-60 768,-60 774,-66 774,-72 774,-72 774,-84 774,-84 774,-90 768,-96 762,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"747\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M321.737,-96.1018C330.735,-101.964 341.381,-107.78 352,-111 404.782,-127.006 420.844,-116 476,-116 476,-116 476,-116 568,-116 623.156,-116 639.218,-127.006 692,-111 699.301,-108.786 706.614,-105.345 713.396,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"715.545,-104.295 722.263,-96.1018 711.902,-98.3176 715.545,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2 -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_block0_branch_bn2</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-76C492,-76 462,-76 462,-76 456,-76 450,-70 450,-64 450,-64 450,-52 450,-52 450,-46 456,-40 462,-40 462,-40 492,-40 492,-40 498,-40 504,-46 504,-52 504,-52 504,-64 504,-64 504,-70 498,-76 492,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-58C422.393,-58 431.311,-58 439.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-61.5001 449.919,-58 439.919,-54.5001 439.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2 -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer1_block0_branch_relu2</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-76C582,-76 552,-76 552,-76 546,-76 540,-70 540,-64 540,-64 540,-52 540,-52 540,-46 546,-40 552,-40 552,-40 582,-40 582,-40 588,-40 594,-46 594,-52 594,-52 594,-64 594,-64 594,-70 588,-76 582,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-58C512.393,-58 521.311,-58 529.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-61.5001 539.919,-58 529.919,-54.5001 529.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2 -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer1_block0_branch_conv2</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M672,-76C672,-76 642,-76 642,-76 636,-76 630,-70 630,-64 630,-64 630,-52 630,-52 630,-46 636,-40 642,-40 642,-40 672,-40 672,-40 678,-40 684,-46 684,-52 684,-52 684,-64 684,-64 684,-70 678,-76 672,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-58C602.393,-58 611.311,-58 619.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-61.5001 629.919,-58 619.919,-54.5001 619.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer1_block0_branch_conv2&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.403,-64.0007C692.481,-65.8366 701.507,-67.8879 710.105,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"709.392,-73.2692 719.919,-72.0725 710.943,-66.4432 709.392,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1 -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer1_block1_bn1</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M852,-96C852,-96 822,-96 822,-96 816,-96 810,-90 810,-84 810,-84 810,-72 810,-72 810,-66 816,-60 822,-60 822,-60 852,-60 852,-60 858,-60 864,-66 864,-72 864,-72 864,-84 864,-84 864,-90 858,-96 852,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_add&#45;&gt;layer1_block1_bn1 -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer1_block0_add&#45;&gt;layer1_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.403,-78C782.393,-78 791.311,-78 799.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"799.919,-81.5001 809.919,-78 799.919,-74.5001 799.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1 -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer1_block1_relu1</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M942,-96C942,-96 912,-96 912,-96 906,-96 900,-90 900,-84 900,-84 900,-72 900,-72 900,-66 906,-60 912,-60 912,-60 942,-60 942,-60 948,-60 954,-66 954,-72 954,-72 954,-84 954,-84 954,-90 948,-96 942,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1&#45;&gt;layer1_block1_relu1 -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer1_block1_bn1&#45;&gt;layer1_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.403,-78C872.393,-78 881.311,-78 889.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"889.919,-81.5001 899.919,-78 889.919,-74.5001 889.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1 -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer1_block1_branch_conv1</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1032,-76C1032,-76 1002,-76 1002,-76 996,-76 990,-70 990,-64 990,-64 990,-52 990,-52 990,-46 996,-40 1002,-40 1002,-40 1032,-40 1032,-40 1038,-40 1044,-46 1044,-52 1044,-52 1044,-64 1044,-64 1044,-70 1038,-76 1032,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1 -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M954.403,-71.9993C962.481,-70.1634 971.507,-68.1121 980.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"980.943,-69.5568 989.919,-63.9275 979.392,-62.7308 980.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_add -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer1_block1_add</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1392,-96C1392,-96 1362,-96 1362,-96 1356,-96 1350,-90 1350,-84 1350,-84 1350,-72 1350,-72 1350,-66 1356,-60 1362,-60 1362,-60 1392,-60 1392,-60 1398,-60 1404,-66 1404,-72 1404,-72 1404,-84 1404,-84 1404,-90 1398,-96 1392,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1377\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M951.737,-96.1018C960.735,-101.964 971.381,-107.78 982,-111 1034.78,-127.006 1050.84,-116 1106,-116 1106,-116 1106,-116 1198,-116 1253.16,-116 1269.22,-127.006 1322,-111 1329.3,-108.786 1336.61,-105.345 1343.4,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1345.55,-104.295 1352.26,-96.1018 1341.9,-98.3176 1345.55,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2 -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer1_block1_branch_bn2</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1122,-76C1122,-76 1092,-76 1092,-76 1086,-76 1080,-70 1080,-64 1080,-64 1080,-52 1080,-52 1080,-46 1086,-40 1092,-40 1092,-40 1122,-40 1122,-40 1128,-40 1134,-46 1134,-52 1134,-52 1134,-64 1134,-64 1134,-70 1128,-76 1122,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2 -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.4,-58C1052.39,-58 1061.31,-58 1069.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1069.92,-61.5001 1079.92,-58 1069.92,-54.5001 1069.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2 -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer1_block1_branch_relu2</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1212,-76C1212,-76 1182,-76 1182,-76 1176,-76 1170,-70 1170,-64 1170,-64 1170,-52 1170,-52 1170,-46 1176,-40 1182,-40 1182,-40 1212,-40 1212,-40 1218,-40 1224,-46 1224,-52 1224,-52 1224,-64 1224,-64 1224,-70 1218,-76 1212,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2 -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1134.4,-58C1142.39,-58 1151.31,-58 1159.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1159.92,-61.5001 1169.92,-58 1159.92,-54.5001 1159.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2 -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer1_block1_branch_conv2</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1302,-76C1302,-76 1272,-76 1272,-76 1266,-76 1260,-70 1260,-64 1260,-64 1260,-52 1260,-52 1260,-46 1266,-40 1272,-40 1272,-40 1302,-40 1302,-40 1308,-40 1314,-46 1314,-52 1314,-52 1314,-64 1314,-64 1314,-70 1308,-76 1302,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1287\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2 -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1224.4,-58C1232.39,-58 1241.31,-58 1249.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1249.92,-61.5001 1259.92,-58 1249.92,-54.5001 1249.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>layer1_block1_branch_conv2&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1314.4,-64.0007C1322.48,-65.8366 1331.51,-67.8879 1340.1,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1339.39,-73.2692 1349.92,-72.0725 1340.94,-66.4432 1339.39,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1 -->\n",
       "<g id=\"node16\" class=\"node\"><title>layer2_block0_bn1</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1486,-96C1486,-96 1456,-96 1456,-96 1450,-96 1444,-90 1444,-84 1444,-84 1444,-72 1444,-72 1444,-66 1450,-60 1456,-60 1456,-60 1486,-60 1486,-60 1492,-60 1498,-66 1498,-72 1498,-72 1498,-84 1498,-84 1498,-90 1492,-96 1486,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1471\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_add&#45;&gt;layer2_block0_bn1 -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>layer1_block1_add&#45;&gt;layer2_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1404.12,-78C1413.34,-78 1423.89,-78 1433.8,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1433.82,-81.5001 1443.82,-78 1433.82,-74.5001 1433.82,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1 -->\n",
       "<g id=\"node17\" class=\"node\"><title>layer2_block0_relu1</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1576,-96C1576,-96 1546,-96 1546,-96 1540,-96 1534,-90 1534,-84 1534,-84 1534,-72 1534,-72 1534,-66 1540,-60 1546,-60 1546,-60 1576,-60 1576,-60 1582,-60 1588,-66 1588,-72 1588,-72 1588,-84 1588,-84 1588,-90 1582,-96 1576,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1561\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1&#45;&gt;layer2_block0_relu1 -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>layer2_block0_bn1&#45;&gt;layer2_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1498.4,-78C1506.39,-78 1515.31,-78 1523.82,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1523.92,-81.5001 1533.92,-78 1523.92,-74.5001 1523.92,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1 -->\n",
       "<g id=\"node18\" class=\"node\"><title>layer2_block0_branch_conv1</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1666,-77C1666,-77 1636,-77 1636,-77 1630,-77 1624,-71 1624,-65 1624,-65 1624,-53 1624,-53 1624,-47 1630,-41 1636,-41 1636,-41 1666,-41 1666,-41 1672,-41 1678,-47 1678,-53 1678,-53 1678,-65 1678,-65 1678,-71 1672,-77 1666,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1651\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1 -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1588.4,-72.2994C1596.48,-70.5552 1605.51,-68.6065 1614.1,-66.7501\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1614.88,-70.1628 1623.92,-64.6311 1613.41,-63.3205 1614.88,-70.1628\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3 -->\n",
       "<g id=\"node22\" class=\"node\"><title>layer2_block0_conv3</title>\n",
       "<g id=\"a_node22\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-152C1936,-152 1906,-152 1906,-152 1900,-152 1894,-146 1894,-140 1894,-140 1894,-128 1894,-128 1894,-122 1900,-116 1906,-116 1906,-116 1936,-116 1936,-116 1942,-116 1948,-122 1948,-128 1948,-128 1948,-140 1948,-140 1948,-146 1942,-152 1936,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-130.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_conv3 -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1585.19,-96.1046C1594.3,-102.243 1605.16,-108.442 1616,-112 1708.77,-142.443 1825.74,-140.431 1883.83,-136.933\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1884.22,-140.415 1893.97,-136.27 1883.76,-133.43 1884.22,-140.415\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2 -->\n",
       "<g id=\"node19\" class=\"node\"><title>layer2_block0_branch_bn2</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1756,-77C1756,-77 1726,-77 1726,-77 1720,-77 1714,-71 1714,-65 1714,-65 1714,-53 1714,-53 1714,-47 1720,-41 1726,-41 1726,-41 1756,-41 1756,-41 1762,-41 1768,-47 1768,-53 1768,-53 1768,-65 1768,-65 1768,-71 1762,-77 1756,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2 -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1678.4,-59C1686.39,-59 1695.31,-59 1703.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1703.92,-62.5001 1713.92,-59 1703.92,-55.5001 1703.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2 -->\n",
       "<g id=\"node20\" class=\"node\"><title>layer2_block0_branch_relu2</title>\n",
       "<g id=\"a_node20\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1846,-77C1846,-77 1816,-77 1816,-77 1810,-77 1804,-71 1804,-65 1804,-65 1804,-53 1804,-53 1804,-47 1810,-41 1816,-41 1816,-41 1846,-41 1846,-41 1852,-41 1858,-47 1858,-53 1858,-53 1858,-65 1858,-65 1858,-71 1852,-77 1846,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1831\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2 -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1768.4,-59C1776.39,-59 1785.31,-59 1793.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1793.92,-62.5001 1803.92,-59 1793.92,-55.5001 1793.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2 -->\n",
       "<g id=\"node21\" class=\"node\"><title>layer2_block0_branch_conv2</title>\n",
       "<g id=\"a_node21\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-77C1936,-77 1906,-77 1906,-77 1900,-77 1894,-71 1894,-65 1894,-65 1894,-53 1894,-53 1894,-47 1900,-41 1906,-41 1906,-41 1936,-41 1936,-41 1942,-41 1948,-47 1948,-53 1948,-53 1948,-65 1948,-65 1948,-71 1942,-77 1936,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2 -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1858.4,-59C1866.39,-59 1875.31,-59 1883.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1883.92,-62.5001 1893.92,-59 1883.92,-55.5001 1883.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_add -->\n",
       "<g id=\"node23\" class=\"node\"><title>layer2_block0_add</title>\n",
       "<g id=\"a_node23\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2026,-97C2026,-97 1996,-97 1996,-97 1990,-97 1984,-91 1984,-85 1984,-85 1984,-73 1984,-73 1984,-67 1990,-61 1996,-61 1996,-61 2026,-61 2026,-61 2032,-61 2038,-67 2038,-73 2038,-73 2038,-85 2038,-85 2038,-91 2032,-97 2026,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2011\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>layer2_block0_branch_conv2&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-65.0007C1956.48,-66.8366 1965.51,-68.8879 1974.1,-70.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1973.39,-74.2692 1983.92,-73.0725 1974.94,-67.4432 1973.39,-74.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>layer2_block0_conv3&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-117.498C1956.83,-112.23 1966.29,-106.317 1975.22,-100.735\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1977.29,-103.569 1983.92,-95.3006 1973.58,-97.6327 1977.29,-103.569\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1 -->\n",
       "<g id=\"node24\" class=\"node\"><title>layer2_block1_bn1</title>\n",
       "<g id=\"a_node24\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2116,-97C2116,-97 2086,-97 2086,-97 2080,-97 2074,-91 2074,-85 2074,-85 2074,-73 2074,-73 2074,-67 2080,-61 2086,-61 2086,-61 2116,-61 2116,-61 2122,-61 2128,-67 2128,-73 2128,-73 2128,-85 2128,-85 2128,-91 2122,-97 2116,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2101\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_add&#45;&gt;layer2_block1_bn1 -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>layer2_block0_add&#45;&gt;layer2_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2038.4,-79C2046.39,-79 2055.31,-79 2063.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2063.92,-82.5001 2073.92,-79 2063.92,-75.5001 2063.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1 -->\n",
       "<g id=\"node25\" class=\"node\"><title>layer2_block1_relu1</title>\n",
       "<g id=\"a_node25\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2206,-97C2206,-97 2176,-97 2176,-97 2170,-97 2164,-91 2164,-85 2164,-85 2164,-73 2164,-73 2164,-67 2170,-61 2176,-61 2176,-61 2206,-61 2206,-61 2212,-61 2218,-67 2218,-73 2218,-73 2218,-85 2218,-85 2218,-91 2212,-97 2206,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2191\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1&#45;&gt;layer2_block1_relu1 -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>layer2_block1_bn1&#45;&gt;layer2_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2128.4,-79C2136.39,-79 2145.31,-79 2153.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2153.92,-82.5001 2163.92,-79 2153.92,-75.5001 2153.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1 -->\n",
       "<g id=\"node26\" class=\"node\"><title>layer2_block1_branch_conv1</title>\n",
       "<g id=\"a_node26\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2296,-82C2296,-82 2266,-82 2266,-82 2260,-82 2254,-76 2254,-70 2254,-70 2254,-58 2254,-58 2254,-52 2260,-46 2266,-46 2266,-46 2296,-46 2296,-46 2302,-46 2308,-52 2308,-58 2308,-58 2308,-70 2308,-70 2308,-76 2302,-82 2296,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2281\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1 -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2218.4,-74.4995C2226.39,-73.1375 2235.31,-71.6175 2243.82,-70.1664\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2244.65,-73.5763 2253.92,-68.4456 2243.47,-66.6758 2244.65,-73.5763\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_add -->\n",
       "<g id=\"node30\" class=\"node\"><title>layer2_block1_add</title>\n",
       "<g id=\"a_node30\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2656,-82C2656,-82 2626,-82 2626,-82 2620,-82 2614,-76 2614,-70 2614,-70 2614,-58 2614,-58 2614,-52 2620,-46 2626,-46 2626,-46 2656,-46 2656,-46 2662,-46 2668,-52 2668,-58 2668,-58 2668,-70 2668,-70 2668,-76 2662,-82 2656,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2641\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge33\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2211.74,-97.0337C2221.35,-104.671 2233.51,-112.769 2246,-117 2298.24,-134.703 2314.84,-122 2370,-122 2370,-122 2370,-122 2462,-122 2517.16,-122 2535.38,-138.893 2586,-117 2599.55,-111.141 2611.6,-100.216 2620.87,-89.7996\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2623.65,-91.9369 2627.39,-82.0259 2618.28,-87.4385 2623.65,-91.9369\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2 -->\n",
       "<g id=\"node27\" class=\"node\"><title>layer2_block1_branch_bn2</title>\n",
       "<g id=\"a_node27\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2386,-82C2386,-82 2356,-82 2356,-82 2350,-82 2344,-76 2344,-70 2344,-70 2344,-58 2344,-58 2344,-52 2350,-46 2356,-46 2356,-46 2386,-46 2386,-46 2392,-46 2398,-52 2398,-58 2398,-58 2398,-70 2398,-70 2398,-76 2392,-82 2386,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2 -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2308.4,-64C2316.39,-64 2325.31,-64 2333.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2333.92,-67.5001 2343.92,-64 2333.92,-60.5001 2333.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2 -->\n",
       "<g id=\"node28\" class=\"node\"><title>layer2_block1_branch_relu2</title>\n",
       "<g id=\"a_node28\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2476,-82C2476,-82 2446,-82 2446,-82 2440,-82 2434,-76 2434,-70 2434,-70 2434,-58 2434,-58 2434,-52 2440,-46 2446,-46 2446,-46 2476,-46 2476,-46 2482,-46 2488,-52 2488,-58 2488,-58 2488,-70 2488,-70 2488,-76 2482,-82 2476,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2461\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2 -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2398.4,-64C2406.39,-64 2415.31,-64 2423.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2423.92,-67.5001 2433.92,-64 2423.92,-60.5001 2423.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2 -->\n",
       "<g id=\"node29\" class=\"node\"><title>layer2_block1_branch_conv2</title>\n",
       "<g id=\"a_node29\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2566,-82C2566,-82 2536,-82 2536,-82 2530,-82 2524,-76 2524,-70 2524,-70 2524,-58 2524,-58 2524,-52 2530,-46 2536,-46 2536,-46 2566,-46 2566,-46 2572,-46 2578,-52 2578,-58 2578,-58 2578,-70 2578,-70 2578,-76 2572,-82 2566,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2551\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2 -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2488.4,-64C2496.39,-64 2505.31,-64 2513.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2513.92,-67.5001 2523.92,-64 2513.92,-60.5001 2513.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge34\" class=\"edge\"><title>layer2_block1_branch_conv2&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2578.4,-64C2586.39,-64 2595.31,-64 2603.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2603.92,-67.5001 2613.92,-64 2603.92,-60.5001 2603.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1 -->\n",
       "<g id=\"node31\" class=\"node\"><title>layer3_block0_bn1</title>\n",
       "<g id=\"a_node31\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2750,-82C2750,-82 2720,-82 2720,-82 2714,-82 2708,-76 2708,-70 2708,-70 2708,-58 2708,-58 2708,-52 2714,-46 2720,-46 2720,-46 2750,-46 2750,-46 2756,-46 2762,-52 2762,-58 2762,-58 2762,-70 2762,-70 2762,-76 2756,-82 2750,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2735\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_add&#45;&gt;layer3_block0_bn1 -->\n",
       "<g id=\"edge35\" class=\"edge\"><title>layer2_block1_add&#45;&gt;layer3_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2668.12,-64C2677.34,-64 2687.89,-64 2697.8,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2697.82,-67.5001 2707.82,-64 2697.82,-60.5001 2697.82,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1 -->\n",
       "<g id=\"node32\" class=\"node\"><title>layer3_block0_relu1</title>\n",
       "<g id=\"a_node32\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2840,-82C2840,-82 2810,-82 2810,-82 2804,-82 2798,-76 2798,-70 2798,-70 2798,-58 2798,-58 2798,-52 2804,-46 2810,-46 2810,-46 2840,-46 2840,-46 2846,-46 2852,-52 2852,-58 2852,-58 2852,-70 2852,-70 2852,-76 2846,-82 2840,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2825\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1&#45;&gt;layer3_block0_relu1 -->\n",
       "<g id=\"edge36\" class=\"edge\"><title>layer3_block0_bn1&#45;&gt;layer3_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2762.4,-64C2770.39,-64 2779.31,-64 2787.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2787.92,-67.5001 2797.92,-64 2787.92,-60.5001 2787.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1 -->\n",
       "<g id=\"node33\" class=\"node\"><title>layer3_block0_branch_conv1</title>\n",
       "<g id=\"a_node33\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2930,-72C2930,-72 2900,-72 2900,-72 2894,-72 2888,-66 2888,-60 2888,-60 2888,-48 2888,-48 2888,-42 2894,-36 2900,-36 2900,-36 2930,-36 2930,-36 2936,-36 2942,-42 2942,-48 2942,-48 2942,-60 2942,-60 2942,-66 2936,-72 2930,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"2915\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1 -->\n",
       "<g id=\"edge37\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2852.4,-60.9997C2860.39,-60.0917 2869.31,-59.0783 2877.82,-58.1109\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2878.38,-61.5706 2887.92,-56.9638 2877.59,-54.6153 2878.38,-61.5706\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3 -->\n",
       "<g id=\"node37\" class=\"node\"><title>layer3_block0_conv3</title>\n",
       "<g id=\"a_node37\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-147C3200,-147 3170,-147 3170,-147 3164,-147 3158,-141 3158,-135 3158,-135 3158,-123 3158,-123 3158,-117 3164,-111 3170,-111 3170,-111 3200,-111 3200,-111 3206,-111 3212,-117 3212,-123 3212,-123 3212,-135 3212,-135 3212,-141 3206,-147 3200,-147\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-125.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_conv3 -->\n",
       "<g id=\"edge41\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2843.22,-82.1953C2853.06,-91.336 2866.16,-101.594 2880,-107 2970.95,-142.522 3088.72,-138.267 3147.4,-133.121\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3148.02,-136.578 3157.65,-132.159 3147.37,-129.609 3148.02,-136.578\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2 -->\n",
       "<g id=\"node34\" class=\"node\"><title>layer3_block0_branch_bn2</title>\n",
       "<g id=\"a_node34\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3020,-72C3020,-72 2990,-72 2990,-72 2984,-72 2978,-66 2978,-60 2978,-60 2978,-48 2978,-48 2978,-42 2984,-36 2990,-36 2990,-36 3020,-36 3020,-36 3026,-36 3032,-42 3032,-48 3032,-48 3032,-60 3032,-60 3032,-66 3026,-72 3020,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2 -->\n",
       "<g id=\"edge38\" class=\"edge\"><title>layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2942.4,-54C2950.39,-54 2959.31,-54 2967.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2967.92,-57.5001 2977.92,-54 2967.92,-50.5001 2967.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2 -->\n",
       "<g id=\"node35\" class=\"node\"><title>layer3_block0_branch_relu2</title>\n",
       "<g id=\"a_node35\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3110,-72C3110,-72 3080,-72 3080,-72 3074,-72 3068,-66 3068,-60 3068,-60 3068,-48 3068,-48 3068,-42 3074,-36 3080,-36 3080,-36 3110,-36 3110,-36 3116,-36 3122,-42 3122,-48 3122,-48 3122,-60 3122,-60 3122,-66 3116,-72 3110,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3095\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2 -->\n",
       "<g id=\"edge39\" class=\"edge\"><title>layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3032.4,-54C3040.39,-54 3049.31,-54 3057.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3057.92,-57.5001 3067.92,-54 3057.92,-50.5001 3057.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2 -->\n",
       "<g id=\"node36\" class=\"node\"><title>layer3_block0_branch_conv2</title>\n",
       "<g id=\"a_node36\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-72C3200,-72 3170,-72 3170,-72 3164,-72 3158,-66 3158,-60 3158,-60 3158,-48 3158,-48 3158,-42 3164,-36 3170,-36 3170,-36 3200,-36 3200,-36 3206,-36 3212,-42 3212,-48 3212,-48 3212,-60 3212,-60 3212,-66 3206,-72 3200,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2 -->\n",
       "<g id=\"edge40\" class=\"edge\"><title>layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3122.4,-54C3130.39,-54 3139.31,-54 3147.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3147.92,-57.5001 3157.92,-54 3147.92,-50.5001 3147.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_add -->\n",
       "<g id=\"node38\" class=\"node\"><title>layer3_block0_add</title>\n",
       "<g id=\"a_node38\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3290,-109C3290,-109 3260,-109 3260,-109 3254,-109 3248,-103 3248,-97 3248,-97 3248,-85 3248,-85 3248,-79 3254,-73 3260,-73 3260,-73 3290,-73 3290,-73 3296,-73 3302,-79 3302,-85 3302,-85 3302,-97 3302,-97 3302,-103 3296,-109 3290,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3275\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge43\" class=\"edge\"><title>layer3_block0_branch_conv2&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-65.1012C3220.66,-68.5716 3229.9,-72.4578 3238.66,-76.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3237.34,-79.3846 3247.92,-80.0341 3240.06,-72.9318 3237.34,-79.3846\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge42\" class=\"edge\"><title>layer3_block0_conv3&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-117.599C3220.66,-114.035 3229.9,-110.043 3238.66,-106.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3240.13,-109.44 3247.92,-102.262 3237.35,-103.013 3240.13,-109.44\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1 -->\n",
       "<g id=\"node39\" class=\"node\"><title>layer3_block1_bn1</title>\n",
       "<g id=\"a_node39\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3380,-109C3380,-109 3350,-109 3350,-109 3344,-109 3338,-103 3338,-97 3338,-97 3338,-85 3338,-85 3338,-79 3344,-73 3350,-73 3350,-73 3380,-73 3380,-73 3386,-73 3392,-79 3392,-85 3392,-85 3392,-97 3392,-97 3392,-103 3386,-109 3380,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3365\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_add&#45;&gt;layer3_block1_bn1 -->\n",
       "<g id=\"edge44\" class=\"edge\"><title>layer3_block0_add&#45;&gt;layer3_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3302.4,-91C3310.39,-91 3319.31,-91 3327.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3327.92,-94.5001 3337.92,-91 3327.92,-87.5001 3327.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1 -->\n",
       "<g id=\"node40\" class=\"node\"><title>layer3_block1_relu1</title>\n",
       "<g id=\"a_node40\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3470,-109C3470,-109 3440,-109 3440,-109 3434,-109 3428,-103 3428,-97 3428,-97 3428,-85 3428,-85 3428,-79 3434,-73 3440,-73 3440,-73 3470,-73 3470,-73 3476,-73 3482,-79 3482,-85 3482,-85 3482,-97 3482,-97 3482,-103 3476,-109 3470,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3455\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1&#45;&gt;layer3_block1_relu1 -->\n",
       "<g id=\"edge45\" class=\"edge\"><title>layer3_block1_bn1&#45;&gt;layer3_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3392.4,-91C3400.39,-91 3409.31,-91 3417.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3417.92,-94.5001 3427.92,-91 3417.92,-87.5001 3417.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1 -->\n",
       "<g id=\"node41\" class=\"node\"><title>layer3_block1_branch_conv1</title>\n",
       "<g id=\"a_node41\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3560,-89C3560,-89 3530,-89 3530,-89 3524,-89 3518,-83 3518,-77 3518,-77 3518,-65 3518,-65 3518,-59 3524,-53 3530,-53 3530,-53 3560,-53 3560,-53 3566,-53 3572,-59 3572,-65 3572,-65 3572,-77 3572,-77 3572,-83 3566,-89 3560,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3545\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1 -->\n",
       "<g id=\"edge46\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3482.4,-84.9993C3490.48,-83.1634 3499.51,-81.1121 3508.1,-79.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3508.94,-82.5568 3517.92,-76.9275 3507.39,-75.7308 3508.94,-82.5568\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_add -->\n",
       "<g id=\"node45\" class=\"node\"><title>layer3_block1_add</title>\n",
       "<g id=\"a_node45\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3920,-109C3920,-109 3890,-109 3890,-109 3884,-109 3878,-103 3878,-97 3878,-97 3878,-85 3878,-85 3878,-79 3884,-73 3890,-73 3890,-73 3920,-73 3920,-73 3926,-73 3932,-79 3932,-85 3932,-85 3932,-97 3932,-97 3932,-103 3926,-109 3920,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3905\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge50\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3479.74,-109.102C3488.74,-114.964 3499.38,-120.78 3510,-124 3562.78,-140.006 3578.84,-129 3634,-129 3634,-129 3634,-129 3726,-129 3781.16,-129 3797.22,-140.006 3850,-124 3857.3,-121.786 3864.61,-118.345 3871.4,-114.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3873.55,-117.295 3880.26,-109.102 3869.9,-111.318 3873.55,-117.295\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2 -->\n",
       "<g id=\"node42\" class=\"node\"><title>layer3_block1_branch_bn2</title>\n",
       "<g id=\"a_node42\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3650,-89C3650,-89 3620,-89 3620,-89 3614,-89 3608,-83 3608,-77 3608,-77 3608,-65 3608,-65 3608,-59 3614,-53 3620,-53 3620,-53 3650,-53 3650,-53 3656,-53 3662,-59 3662,-65 3662,-65 3662,-77 3662,-77 3662,-83 3656,-89 3650,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2 -->\n",
       "<g id=\"edge47\" class=\"edge\"><title>layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3572.4,-71C3580.39,-71 3589.31,-71 3597.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3597.92,-74.5001 3607.92,-71 3597.92,-67.5001 3597.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2 -->\n",
       "<g id=\"node43\" class=\"node\"><title>layer3_block1_branch_relu2</title>\n",
       "<g id=\"a_node43\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3740,-89C3740,-89 3710,-89 3710,-89 3704,-89 3698,-83 3698,-77 3698,-77 3698,-65 3698,-65 3698,-59 3704,-53 3710,-53 3710,-53 3740,-53 3740,-53 3746,-53 3752,-59 3752,-65 3752,-65 3752,-77 3752,-77 3752,-83 3746,-89 3740,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3725\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2 -->\n",
       "<g id=\"edge48\" class=\"edge\"><title>layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3662.4,-71C3670.39,-71 3679.31,-71 3687.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3687.92,-74.5001 3697.92,-71 3687.92,-67.5001 3687.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2 -->\n",
       "<g id=\"node44\" class=\"node\"><title>layer3_block1_branch_conv2</title>\n",
       "<g id=\"a_node44\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3830,-89C3830,-89 3800,-89 3800,-89 3794,-89 3788,-83 3788,-77 3788,-77 3788,-65 3788,-65 3788,-59 3794,-53 3800,-53 3800,-53 3830,-53 3830,-53 3836,-53 3842,-59 3842,-65 3842,-65 3842,-77 3842,-77 3842,-83 3836,-89 3830,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3815\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2 -->\n",
       "<g id=\"edge49\" class=\"edge\"><title>layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3752.4,-71C3760.39,-71 3769.31,-71 3777.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3777.92,-74.5001 3787.92,-71 3777.92,-67.5001 3777.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge51\" class=\"edge\"><title>layer3_block1_branch_conv2&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3842.4,-77.0007C3850.48,-78.8366 3859.51,-80.8879 3868.1,-82.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3867.39,-86.2692 3877.92,-85.0725 3868.94,-79.4432 3867.39,-86.2692\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1 -->\n",
       "<g id=\"node46\" class=\"node\"><title>layer4_block0_bn1</title>\n",
       "<g id=\"a_node46\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4014,-109C4014,-109 3984,-109 3984,-109 3978,-109 3972,-103 3972,-97 3972,-97 3972,-85 3972,-85 3972,-79 3978,-73 3984,-73 3984,-73 4014,-73 4014,-73 4020,-73 4026,-79 4026,-85 4026,-85 4026,-97 4026,-97 4026,-103 4020,-109 4014,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3999\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_add&#45;&gt;layer4_block0_bn1 -->\n",
       "<g id=\"edge52\" class=\"edge\"><title>layer3_block1_add&#45;&gt;layer4_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3932.12,-91C3941.34,-91 3951.89,-91 3961.8,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3961.82,-94.5001 3971.82,-91 3961.82,-87.5001 3961.82,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1 -->\n",
       "<g id=\"node47\" class=\"node\"><title>layer4_block0_relu1</title>\n",
       "<g id=\"a_node47\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4104,-109C4104,-109 4074,-109 4074,-109 4068,-109 4062,-103 4062,-97 4062,-97 4062,-85 4062,-85 4062,-79 4068,-73 4074,-73 4074,-73 4104,-73 4104,-73 4110,-73 4116,-79 4116,-85 4116,-85 4116,-97 4116,-97 4116,-103 4110,-109 4104,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"4089\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1&#45;&gt;layer4_block0_relu1 -->\n",
       "<g id=\"edge53\" class=\"edge\"><title>layer4_block0_bn1&#45;&gt;layer4_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4026.4,-91C4034.39,-91 4043.31,-91 4051.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4051.92,-94.5001 4061.92,-91 4051.92,-87.5001 4051.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1 -->\n",
       "<g id=\"node48\" class=\"node\"><title>layer4_block0_branch_conv1</title>\n",
       "<g id=\"a_node48\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4194,-68C4194,-68 4164,-68 4164,-68 4158,-68 4152,-62 4152,-56 4152,-56 4152,-44 4152,-44 4152,-38 4158,-32 4164,-32 4164,-32 4194,-32 4194,-32 4200,-32 4206,-38 4206,-44 4206,-44 4206,-56 4206,-56 4206,-62 4200,-68 4194,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4179\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1 -->\n",
       "<g id=\"edge54\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.4,-78.6986C4124.66,-74.8531 4133.9,-70.5468 4142.66,-66.463\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4144.33,-69.5472 4151.92,-62.1514 4141.38,-63.2021 4144.33,-69.5472\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3 -->\n",
       "<g id=\"node52\" class=\"node\"><title>layer4_block0_conv3</title>\n",
       "<g id=\"a_node52\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-143C4464,-143 4434,-143 4434,-143 4428,-143 4422,-137 4422,-131 4422,-131 4422,-119 4422,-119 4422,-113 4428,-107 4434,-107 4434,-107 4464,-107 4464,-107 4470,-107 4476,-113 4476,-119 4476,-119 4476,-131 4476,-131 4476,-137 4470,-143 4464,-143\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-121.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_conv3 -->\n",
       "<g id=\"edge58\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.22,-97.6371C4125,-99.6421 4134.88,-101.664 4144,-103 4239.91,-117.051 4354.23,-122.258 4411.49,-124.083\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.73,-127.592 4421.83,-124.395 4411.94,-120.595 4411.73,-127.592\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2 -->\n",
       "<g id=\"node49\" class=\"node\"><title>layer4_block0_branch_bn2</title>\n",
       "<g id=\"a_node49\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4284,-68C4284,-68 4254,-68 4254,-68 4248,-68 4242,-62 4242,-56 4242,-56 4242,-44 4242,-44 4242,-38 4248,-32 4254,-32 4254,-32 4284,-32 4284,-32 4290,-32 4296,-38 4296,-44 4296,-44 4296,-56 4296,-56 4296,-62 4290,-68 4284,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2 -->\n",
       "<g id=\"edge55\" class=\"edge\"><title>layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4206.4,-50C4214.39,-50 4223.31,-50 4231.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4231.92,-53.5001 4241.92,-50 4231.92,-46.5001 4231.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2 -->\n",
       "<g id=\"node50\" class=\"node\"><title>layer4_block0_branch_relu2</title>\n",
       "<g id=\"a_node50\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4374,-68C4374,-68 4344,-68 4344,-68 4338,-68 4332,-62 4332,-56 4332,-56 4332,-44 4332,-44 4332,-38 4338,-32 4344,-32 4344,-32 4374,-32 4374,-32 4380,-32 4386,-38 4386,-44 4386,-44 4386,-56 4386,-56 4386,-62 4380,-68 4374,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4359\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2 -->\n",
       "<g id=\"edge56\" class=\"edge\"><title>layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4296.4,-50C4304.39,-50 4313.31,-50 4321.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4321.92,-53.5001 4331.92,-50 4321.92,-46.5001 4321.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2 -->\n",
       "<g id=\"node51\" class=\"node\"><title>layer4_block0_branch_conv2</title>\n",
       "<g id=\"a_node51\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-68C4464,-68 4434,-68 4434,-68 4428,-68 4422,-62 4422,-56 4422,-56 4422,-44 4422,-44 4422,-38 4428,-32 4434,-32 4434,-32 4464,-32 4464,-32 4470,-32 4476,-38 4476,-44 4476,-44 4476,-56 4476,-56 4476,-62 4470,-68 4464,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2 -->\n",
       "<g id=\"edge57\" class=\"edge\"><title>layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4386.4,-50C4394.39,-50 4403.31,-50 4411.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.92,-53.5001 4421.92,-50 4411.92,-46.5001 4411.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_add -->\n",
       "<g id=\"node53\" class=\"node\"><title>layer4_block0_add</title>\n",
       "<g id=\"a_node53\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M4554,-105C4554,-105 4524,-105 4524,-105 4518,-105 4512,-99 4512,-93 4512,-93 4512,-81 4512,-81 4512,-75 4518,-69 4524,-69 4524,-69 4554,-69 4554,-69 4560,-69 4566,-75 4566,-81 4566,-81 4566,-93 4566,-93 4566,-99 4560,-105 4554,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4539\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge60\" class=\"edge\"><title>layer4_block0_branch_conv2&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-61.1012C4484.66,-64.5716 4493.9,-68.4578 4502.66,-72.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4501.34,-75.3846 4511.92,-76.0341 4504.06,-68.9318 4501.34,-75.3846\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge59\" class=\"edge\"><title>layer4_block0_conv3&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-113.599C4484.66,-110.035 4493.9,-106.043 4502.66,-102.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4504.13,-105.44 4511.92,-98.2623 4501.35,-99.0135 4504.13,-105.44\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1 -->\n",
       "<g id=\"node54\" class=\"node\"><title>layer4_block1_bn1</title>\n",
       "<g id=\"a_node54\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4644,-105C4644,-105 4614,-105 4614,-105 4608,-105 4602,-99 4602,-93 4602,-93 4602,-81 4602,-81 4602,-75 4608,-69 4614,-69 4614,-69 4644,-69 4644,-69 4650,-69 4656,-75 4656,-81 4656,-81 4656,-93 4656,-93 4656,-99 4650,-105 4644,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4629\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_add&#45;&gt;layer4_block1_bn1 -->\n",
       "<g id=\"edge61\" class=\"edge\"><title>layer4_block0_add&#45;&gt;layer4_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4566.4,-87C4574.39,-87 4583.31,-87 4591.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4591.92,-90.5001 4601.92,-87 4591.92,-83.5001 4591.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1 -->\n",
       "<g id=\"node55\" class=\"node\"><title>layer4_block1_relu1</title>\n",
       "<g id=\"a_node55\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4734,-105C4734,-105 4704,-105 4704,-105 4698,-105 4692,-99 4692,-93 4692,-93 4692,-81 4692,-81 4692,-75 4698,-69 4704,-69 4704,-69 4734,-69 4734,-69 4740,-69 4746,-75 4746,-81 4746,-81 4746,-93 4746,-93 4746,-99 4740,-105 4734,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4719\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1&#45;&gt;layer4_block1_relu1 -->\n",
       "<g id=\"edge62\" class=\"edge\"><title>layer4_block1_bn1&#45;&gt;layer4_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4656.4,-87C4664.39,-87 4673.31,-87 4681.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4681.92,-90.5001 4691.92,-87 4681.92,-83.5001 4681.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1 -->\n",
       "<g id=\"node56\" class=\"node\"><title>layer4_block1_branch_conv1</title>\n",
       "<g id=\"a_node56\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4824,-85C4824,-85 4794,-85 4794,-85 4788,-85 4782,-79 4782,-73 4782,-73 4782,-61 4782,-61 4782,-55 4788,-49 4794,-49 4794,-49 4824,-49 4824,-49 4830,-49 4836,-55 4836,-61 4836,-61 4836,-73 4836,-73 4836,-79 4830,-85 4824,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4809\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1 -->\n",
       "<g id=\"edge63\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4746.4,-80.9993C4754.48,-79.1634 4763.51,-77.1121 4772.1,-75.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4772.94,-78.5568 4781.92,-72.9275 4771.39,-71.7308 4772.94,-78.5568\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_add -->\n",
       "<g id=\"node60\" class=\"node\"><title>layer4_block1_add</title>\n",
       "<g id=\"a_node60\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M5184,-105C5184,-105 5154,-105 5154,-105 5148,-105 5142,-99 5142,-93 5142,-93 5142,-81 5142,-81 5142,-75 5148,-69 5154,-69 5154,-69 5184,-69 5184,-69 5190,-69 5196,-75 5196,-81 5196,-81 5196,-93 5196,-93 5196,-99 5190,-105 5184,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5169\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge67\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4743.74,-105.102C4752.74,-110.964 4763.38,-116.78 4774,-120 4826.78,-136.006 4842.84,-125 4898,-125 4898,-125 4898,-125 4990,-125 5045.16,-125 5061.22,-136.006 5114,-120 5121.3,-117.786 5128.61,-114.345 5135.4,-110.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5137.55,-113.295 5144.26,-105.102 5133.9,-107.318 5137.55,-113.295\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2 -->\n",
       "<g id=\"node57\" class=\"node\"><title>layer4_block1_branch_bn2</title>\n",
       "<g id=\"a_node57\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4914,-85C4914,-85 4884,-85 4884,-85 4878,-85 4872,-79 4872,-73 4872,-73 4872,-61 4872,-61 4872,-55 4878,-49 4884,-49 4884,-49 4914,-49 4914,-49 4920,-49 4926,-55 4926,-61 4926,-61 4926,-73 4926,-73 4926,-79 4920,-85 4914,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2 -->\n",
       "<g id=\"edge64\" class=\"edge\"><title>layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4836.4,-67C4844.39,-67 4853.31,-67 4861.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4861.92,-70.5001 4871.92,-67 4861.92,-63.5001 4861.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2 -->\n",
       "<g id=\"node58\" class=\"node\"><title>layer4_block1_branch_relu2</title>\n",
       "<g id=\"a_node58\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M5004,-85C5004,-85 4974,-85 4974,-85 4968,-85 4962,-79 4962,-73 4962,-73 4962,-61 4962,-61 4962,-55 4968,-49 4974,-49 4974,-49 5004,-49 5004,-49 5010,-49 5016,-55 5016,-61 5016,-61 5016,-73 5016,-73 5016,-79 5010,-85 5004,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4989\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2 -->\n",
       "<g id=\"edge65\" class=\"edge\"><title>layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4926.4,-67C4934.39,-67 4943.31,-67 4951.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4951.92,-70.5001 4961.92,-67 4951.92,-63.5001 4951.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2 -->\n",
       "<g id=\"node59\" class=\"node\"><title>layer4_block1_branch_conv2</title>\n",
       "<g id=\"a_node59\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M5094,-85C5094,-85 5064,-85 5064,-85 5058,-85 5052,-79 5052,-73 5052,-73 5052,-61 5052,-61 5052,-55 5058,-49 5064,-49 5064,-49 5094,-49 5094,-49 5100,-49 5106,-55 5106,-61 5106,-61 5106,-73 5106,-73 5106,-79 5100,-85 5094,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"5079\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2 -->\n",
       "<g id=\"edge66\" class=\"edge\"><title>layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5016.4,-67C5024.39,-67 5033.31,-67 5041.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5041.92,-70.5001 5051.92,-67 5041.92,-63.5001 5041.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge68\" class=\"edge\"><title>layer4_block1_branch_conv2&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5106.4,-73.0007C5114.48,-74.8366 5123.51,-76.8879 5132.1,-78.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5131.39,-82.2692 5141.92,-81.0725 5132.94,-75.4432 5131.39,-82.2692\"/>\n",
       "</g>\n",
       "<!-- final_in -->\n",
       "<g id=\"node61\" class=\"node\"><title>final_in</title>\n",
       "<g id=\"a_node61\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5274,-105C5274,-105 5244,-105 5244,-105 5238,-105 5232,-99 5232,-93 5232,-93 5232,-81 5232,-81 5232,-75 5238,-69 5244,-69 5244,-69 5274,-69 5274,-69 5280,-69 5286,-75 5286,-81 5286,-81 5286,-93 5286,-93 5286,-99 5280,-105 5274,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5259\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">in</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_add&#45;&gt;final_in -->\n",
       "<g id=\"edge69\" class=\"edge\"><title>layer4_block1_add&#45;&gt;final_in</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5196.4,-87C5204.39,-87 5213.31,-87 5221.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5221.92,-90.5001 5231.92,-87 5221.92,-83.5001 5221.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_maxpool -->\n",
       "<g id=\"node62\" class=\"node\"><title>final_maxpool</title>\n",
       "<g id=\"a_node62\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#fdb462\" stroke=\"black\" d=\"M5375,-78C5375,-78 5334,-78 5334,-78 5328,-78 5322,-72 5322,-66 5322,-66 5322,-54 5322,-54 5322,-48 5328,-42 5334,-42 5334,-42 5375,-42 5375,-42 5381,-42 5387,-48 5387,-54 5387,-54 5387,-66 5387,-66 5387,-72 5381,-78 5375,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">maxpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_maxpool -->\n",
       "<g id=\"edge70\" class=\"edge\"><title>final_in&#45;&gt;final_maxpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-79.479C5294.13,-77.1436 5303.26,-74.509 5312.12,-71.9497\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.31,-75.2492 5321.94,-69.1121 5311.37,-68.524 5313.31,-75.2492\"/>\n",
       "</g>\n",
       "<!-- final_avgpool -->\n",
       "<g id=\"node63\" class=\"node\"><title>final_avgpool</title>\n",
       "<g id=\"a_node63\"><a xlink:title=\"AvgPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;ceil_mode&#39;: False, &#39;count_include_pad&#39;: True}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M5372.5,-132C5372.5,-132 5336.5,-132 5336.5,-132 5330.5,-132 5324.5,-126 5324.5,-120 5324.5,-120 5324.5,-108 5324.5,-108 5324.5,-102 5330.5,-96 5336.5,-96 5336.5,-96 5372.5,-96 5372.5,-96 5378.5,-96 5384.5,-102 5384.5,-108 5384.5,-108 5384.5,-120 5384.5,-120 5384.5,-126 5378.5,-132 5372.5,-132\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-110.3\" font-family=\"Times,serif\" font-size=\"14.00\">avgpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_avgpool -->\n",
       "<g id=\"edge71\" class=\"edge\"><title>final_in&#45;&gt;final_avgpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-94.521C5294.87,-97.0688 5304.92,-99.9726 5314.53,-102.746\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.73,-106.157 5324.3,-105.569 5315.67,-99.4322 5313.73,-106.157\"/>\n",
       "</g>\n",
       "<!-- final_concat -->\n",
       "<g id=\"node64\" class=\"node\"><title>final_concat</title>\n",
       "<g id=\"a_node64\"><a xlink:title=\"Concat {}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M5465,-105C5465,-105 5435,-105 5435,-105 5429,-105 5423,-99 5423,-93 5423,-93 5423,-81 5423,-81 5423,-75 5429,-69 5435,-69 5435,-69 5465,-69 5465,-69 5471,-69 5477,-75 5477,-81 5477,-81 5477,-93 5477,-93 5477,-99 5471,-105 5465,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5450\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">concat</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_maxpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge72\" class=\"edge\"><title>final_maxpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5387.15,-69.141C5395.53,-71.5601 5404.63,-74.1884 5413.21,-76.6659\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5412.39,-80.0717 5422.97,-79.4835 5414.33,-73.3465 5412.39,-80.0717\"/>\n",
       "</g>\n",
       "<!-- final_avgpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge73\" class=\"edge\"><title>final_avgpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5384.57,-105.605C5393.55,-103.013 5403.54,-100.127 5412.93,-97.4155\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5414.17,-100.702 5422.8,-94.5648 5412.22,-93.9766 5414.17,-100.702\"/>\n",
       "</g>\n",
       "<!-- final_flatten -->\n",
       "<g id=\"node65\" class=\"node\"><title>final_flatten</title>\n",
       "<g id=\"a_node65\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M5555,-105C5555,-105 5525,-105 5525,-105 5519,-105 5513,-99 5513,-93 5513,-93 5513,-81 5513,-81 5513,-75 5519,-69 5525,-69 5525,-69 5555,-69 5555,-69 5561,-69 5567,-75 5567,-81 5567,-81 5567,-93 5567,-93 5567,-99 5561,-105 5555,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5540\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_concat&#45;&gt;final_flatten -->\n",
       "<g id=\"edge74\" class=\"edge\"><title>final_concat&#45;&gt;final_flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5477.4,-87C5485.39,-87 5494.31,-87 5502.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5502.92,-90.5001 5512.92,-87 5502.92,-83.5001 5502.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_linear -->\n",
       "<g id=\"node66\" class=\"node\"><title>final_linear</title>\n",
       "<g id=\"a_node66\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: Parameter containing:\\ntensor([&#45;0.0026, &#45;0.0122, &#160;0.0107, &#160;0.0264, &#45;0.0252, &#160;0.0293, &#160;0.0281, &#45;0.0416,\\n &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#45;0.0281, &#45;0.0249], requires_grad=True)}\">\n",
       "<path fill=\"#ccebc5\" stroke=\"black\" d=\"M5645,-105C5645,-105 5615,-105 5615,-105 5609,-105 5603,-99 5603,-93 5603,-93 5603,-81 5603,-81 5603,-75 5609,-69 5615,-69 5615,-69 5645,-69 5645,-69 5651,-69 5657,-75 5657,-81 5657,-81 5657,-93 5657,-93 5657,-99 5651,-105 5645,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5630\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_flatten&#45;&gt;final_linear -->\n",
       "<g id=\"edge75\" class=\"edge\"><title>final_flatten&#45;&gt;final_linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5567.4,-87C5575.39,-87 5584.31,-87 5592.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5592.92,-90.5001 5602.92,-87 5592.92,-83.5001 5592.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node67\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node67\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5746,-105C5746,-105 5705,-105 5705,-105 5699,-105 5693,-99 5693,-93 5693,-93 5693,-81 5693,-81 5693,-75 5699,-69 5705,-69 5705,-69 5746,-69 5746,-69 5752,-69 5758,-75 5758,-81 5758,-81 5758,-93 5758,-93 5758,-99 5752,-105 5746,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5725.5\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_linear&#45;&gt;classifier -->\n",
       "<g id=\"edge76\" class=\"edge\"><title>final_linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5657.04,-87C5665.05,-87 5674.06,-87 5682.83,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5682.95,-90.5001 5692.94,-87 5682.94,-83.5001 5682.95,-90.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node68\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-96C42,-96 12,-96 12,-96 6,-96 0,-90 0,-84 0,-84 0,-72 0,-72 0,-66 6,-60 12,-60 12,-60 42,-60 42,-60 48,-60 54,-66 54,-72 54,-72 54,-84 54,-84 54,-90 48,-96 42,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-78C62.3932,-78 71.3106,-78 79.8241,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-81.5001 89.919,-78 79.919,-74.5001 79.919,-81.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7f0103da4438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.0293       4.4231       1.7187       0.3601       0.3057       1.2499       0.5484       4.7288\n",
      "           2       0.0587       4.4285       1.0809       0.6089       0.3075       1.2263       0.6002       9.4648\n",
      "           3       0.0880       4.4296       0.7759       0.7262       0.3065       0.8122       0.7257      14.2008\n",
      "           4       0.1173       4.4305       0.6356       0.7760       0.3067       0.7249       0.7537      18.9380\n",
      "           5       0.1467       4.4762       0.5627       0.8032       0.3065       0.8471       0.7234      23.7208\n",
      "           6       0.1760       4.4272       0.4928       0.8311       0.3057       0.6922       0.7658      28.4537\n",
      "           7       0.2053       4.4273       0.4557       0.8422       0.3058       0.5797       0.8057      33.1868\n",
      "           8       0.2347       4.4266       0.4255       0.8520       0.3060       0.7200       0.7643      37.9194\n",
      "           9       0.2640       4.4507       0.4046       0.8603       0.3052       0.5625       0.8094      42.6752\n",
      "          10       0.2933       4.4258       0.3878       0.8666       0.3041       0.5470       0.8131      47.4052\n",
      "          11       0.3227       4.4209       0.3824       0.8681       0.3055       0.6970       0.7672      52.1316\n",
      "          12       0.3520       4.4157       0.3845       0.8657       0.3056       0.5381       0.8128      56.8529\n",
      "          13       0.3813       4.4475       0.3840       0.8693       0.3055       0.4634       0.8406      61.6059\n",
      "          14       0.4107       4.4159       0.3860       0.8660       0.3049       0.5881       0.8019      66.3266\n",
      "          15       0.4400       4.4134       0.3823       0.8676       0.3057       0.9815       0.6732      71.0457\n",
      "          16       0.4110       4.4093       0.3788       0.8690       0.3054       0.5483       0.8149      75.7605\n",
      "          17       0.3820       4.4413       0.3498       0.8786       0.3046       0.4391       0.8504      80.5064\n",
      "          18       0.3530       4.4051       0.3368       0.8821       0.3044       0.5289       0.8188      85.2159\n",
      "          19       0.3240       4.4090       0.3119       0.8929       0.3038       0.4400       0.8524      89.9287\n",
      "          20       0.2950       4.4063       0.2894       0.9012       0.3049       0.4160       0.8645      94.6399\n",
      "          21       0.2660       4.4242       0.2726       0.9058       0.3058       0.5180       0.8251      99.3699\n",
      "          22       0.2370       4.4034       0.2539       0.9119       0.3046       0.3544       0.8778     104.0779\n",
      "          23       0.2080       4.4029       0.2328       0.9197       0.3046       0.4088       0.8640     108.7854\n",
      "          24       0.1790       4.4040       0.2114       0.9272       0.3048       0.4448       0.8564     113.4942\n",
      "          25       0.1500       4.4145       0.1900       0.9337       0.3052       0.3252       0.8934     118.2139\n",
      "          26       0.1210       4.4036       0.1609       0.9445       0.3052       0.3349       0.8916     122.9228\n",
      "          27       0.0920       4.4054       0.1324       0.9537       0.3048       0.2900       0.9063     127.6329\n",
      "          28       0.0630       4.4079       0.1070       0.9630       0.3050       0.2461       0.9213     132.3458\n",
      "          29       0.0340       4.4344       0.0743       0.9747       0.3066       0.2196       0.9287     137.0868\n",
      "          30       0.0050       4.4051       0.0503       0.9839       0.3044       0.2042       0.9357     141.7963\n",
      "          31       0.0040       4.4052       0.0395       0.9880       0.3026       0.2001       0.9359     146.5041\n",
      "          32       0.0030       4.4019       0.0368       0.9886       0.3431       0.1997       0.9374     151.2490\n",
      "          33       0.0020       4.4019       0.0357       0.9897       0.3050       0.1988       0.9363     155.9558\n",
      "          34       0.0010       4.4027       0.0343       0.9897       0.3047       0.1992       0.9370     160.6633\n",
      "          35       0.0000       4.4045       0.0367       0.9889       0.3043       0.1985       0.9370     165.3720\n"
     ]
    }
   ],
   "source": [
    "lr_schedule = PiecewiseLinear([0, 15, 30, 35], [0, 0.44, 0.005, 0])\n",
    "batch_size = 512\n",
    "\n",
    "n = DAWN_net()\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR()])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 3: Regularisation](https://www.myrtle.ai/2018/09/24/how_to_train_your_resnet_3/) - cutout+30 epochs+batch_size=512 (161s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"43pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 43.08\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.187305 0.187305) rotate(0) translate(4 226)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-226 5762,-226 5762,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-52C94,-52 140,-52 140,-52 146,-52 152,-58 152,-64 152,-64 152,-115 152,-115 152,-121 146,-127 140,-127 140,-127 94,-127 94,-127 88,-127 82,-121 82,-115 82,-115 82,-64 82,-64 82,-58 88,-52 94,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M176,-16C176,-16 1408,-16 1408,-16 1414,-16 1420,-22 1420,-28 1420,-28 1420,-167 1420,-167 1420,-173 1414,-179 1408,-179 1408,-179 176,-179 176,-179 170,-179 164,-173 164,-167 164,-167 164,-28 164,-28 164,-22 170,-16 176,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"792\" y=\"-163.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer1_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M184,-24C184,-24 770,-24 770,-24 776,-24 782,-30 782,-36 782,-36 782,-136 782,-136 782,-142 776,-148 770,-148 770,-148 184,-148 184,-148 178,-148 172,-142 172,-136 172,-136 172,-36 172,-36 172,-30 178,-24 184,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer1_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-32C364,-32 680,-32 680,-32 686,-32 692,-38 692,-44 692,-44 692,-95 692,-95 692,-101 686,-107 680,-107 680,-107 364,-107 364,-107 358,-107 352,-101 352,-95 352,-95 352,-44 352,-44 352,-38 358,-32 364,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer1_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M814,-24C814,-24 1400,-24 1400,-24 1406,-24 1412,-30 1412,-36 1412,-36 1412,-136 1412,-136 1412,-142 1406,-148 1400,-148 1400,-148 814,-148 814,-148 808,-148 802,-142 802,-136 802,-136 802,-36 802,-36 802,-30 808,-24 814,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer1_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M994,-32C994,-32 1310,-32 1310,-32 1316,-32 1322,-38 1322,-44 1322,-44 1322,-95 1322,-95 1322,-101 1316,-107 1310,-107 1310,-107 994,-107 994,-107 988,-107 982,-101 982,-95 982,-95 982,-44 982,-44 982,-38 988,-32 994,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"1152\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1440,-17C1440,-17 2672,-17 2672,-17 2678,-17 2684,-23 2684,-29 2684,-29 2684,-202 2684,-202 2684,-208 2678,-214 2672,-214 2672,-214 1440,-214 1440,-214 1434,-214 1428,-208 1428,-202 1428,-202 1428,-29 1428,-29 1428,-23 1434,-17 1440,-17\"/>\n",
       "<text text-anchor=\"middle\" x=\"2056\" y=\"-198.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_layer2_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1448,-25C1448,-25 2034,-25 2034,-25 2040,-25 2046,-31 2046,-37 2046,-37 2046,-171 2046,-171 2046,-177 2040,-183 2034,-183 2034,-183 1448,-183 1448,-183 1442,-183 1436,-177 1436,-171 1436,-171 1436,-37 1436,-37 1436,-31 1442,-25 1448,-25\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-167.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_layer2_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1628,-33C1628,-33 1944,-33 1944,-33 1950,-33 1956,-39 1956,-45 1956,-45 1956,-96 1956,-96 1956,-102 1950,-108 1944,-108 1944,-108 1628,-108 1628,-108 1622,-108 1616,-102 1616,-96 1616,-96 1616,-45 1616,-45 1616,-39 1622,-33 1628,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"1786\" y=\"-92.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_layer2_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2078,-30C2078,-30 2664,-30 2664,-30 2670,-30 2676,-36 2676,-42 2676,-42 2676,-142 2676,-142 2676,-148 2670,-154 2664,-154 2664,-154 2078,-154 2078,-154 2072,-154 2066,-148 2066,-142 2066,-142 2066,-42 2066,-42 2066,-36 2072,-30 2078,-30\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-138.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust11\" class=\"cluster\"><title>cluster_layer2_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2258,-38C2258,-38 2574,-38 2574,-38 2580,-38 2586,-44 2586,-50 2586,-50 2586,-101 2586,-101 2586,-107 2580,-113 2574,-113 2574,-113 2258,-113 2258,-113 2252,-113 2246,-107 2246,-101 2246,-101 2246,-50 2246,-50 2246,-44 2252,-38 2258,-38\"/>\n",
       "<text text-anchor=\"middle\" x=\"2416\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust12\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2704,-12C2704,-12 3936,-12 3936,-12 3942,-12 3948,-18 3948,-24 3948,-24 3948,-197 3948,-197 3948,-203 3942,-209 3936,-209 3936,-209 2704,-209 2704,-209 2698,-209 2692,-203 2692,-197 2692,-197 2692,-24 2692,-24 2692,-18 2698,-12 2704,-12\"/>\n",
       "<text text-anchor=\"middle\" x=\"3320\" y=\"-193.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust13\" class=\"cluster\"><title>cluster_layer3_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2712,-20C2712,-20 3298,-20 3298,-20 3304,-20 3310,-26 3310,-32 3310,-32 3310,-166 3310,-166 3310,-172 3304,-178 3298,-178 3298,-178 2712,-178 2712,-178 2706,-178 2700,-172 2700,-166 2700,-166 2700,-32 2700,-32 2700,-26 2706,-20 2712,-20\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-162.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust14\" class=\"cluster\"><title>cluster_layer3_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2892,-28C2892,-28 3208,-28 3208,-28 3214,-28 3220,-34 3220,-40 3220,-40 3220,-91 3220,-91 3220,-97 3214,-103 3208,-103 3208,-103 2892,-103 2892,-103 2886,-103 2880,-97 2880,-91 2880,-91 2880,-40 2880,-40 2880,-34 2886,-28 2892,-28\"/>\n",
       "<text text-anchor=\"middle\" x=\"3050\" y=\"-87.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust15\" class=\"cluster\"><title>cluster_layer3_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3342,-37C3342,-37 3928,-37 3928,-37 3934,-37 3940,-43 3940,-49 3940,-49 3940,-149 3940,-149 3940,-155 3934,-161 3928,-161 3928,-161 3342,-161 3342,-161 3336,-161 3330,-155 3330,-149 3330,-149 3330,-49 3330,-49 3330,-43 3336,-37 3342,-37\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-145.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust16\" class=\"cluster\"><title>cluster_layer3_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3522,-45C3522,-45 3838,-45 3838,-45 3844,-45 3850,-51 3850,-57 3850,-57 3850,-108 3850,-108 3850,-114 3844,-120 3838,-120 3838,-120 3522,-120 3522,-120 3516,-120 3510,-114 3510,-108 3510,-108 3510,-57 3510,-57 3510,-51 3516,-45 3522,-45\"/>\n",
       "<text text-anchor=\"middle\" x=\"3680\" y=\"-104.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust17\" class=\"cluster\"><title>cluster_layer4</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3968,-8C3968,-8 5200,-8 5200,-8 5206,-8 5212,-14 5212,-20 5212,-20 5212,-193 5212,-193 5212,-199 5206,-205 5200,-205 5200,-205 3968,-205 3968,-205 3962,-205 3956,-199 3956,-193 3956,-193 3956,-20 3956,-20 3956,-14 3962,-8 3968,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"4584\" y=\"-189.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer4</text>\n",
       "</g>\n",
       "<g id=\"clust18\" class=\"cluster\"><title>cluster_layer4_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3976,-16C3976,-16 4562,-16 4562,-16 4568,-16 4574,-22 4574,-28 4574,-28 4574,-162 4574,-162 4574,-168 4568,-174 4562,-174 4562,-174 3976,-174 3976,-174 3970,-174 3964,-168 3964,-162 3964,-162 3964,-28 3964,-28 3964,-22 3970,-16 3976,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-158.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust19\" class=\"cluster\"><title>cluster_layer4_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4156,-24C4156,-24 4472,-24 4472,-24 4478,-24 4484,-30 4484,-36 4484,-36 4484,-87 4484,-87 4484,-93 4478,-99 4472,-99 4472,-99 4156,-99 4156,-99 4150,-99 4144,-93 4144,-87 4144,-87 4144,-36 4144,-36 4144,-30 4150,-24 4156,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"4314\" y=\"-83.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust20\" class=\"cluster\"><title>cluster_layer4_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4606,-33C4606,-33 5192,-33 5192,-33 5198,-33 5204,-39 5204,-45 5204,-45 5204,-145 5204,-145 5204,-151 5198,-157 5192,-157 5192,-157 4606,-157 4606,-157 4600,-157 4594,-151 4594,-145 4594,-145 4594,-45 4594,-45 4594,-39 4600,-33 4606,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-141.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust21\" class=\"cluster\"><title>cluster_layer4_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4786,-41C4786,-41 5102,-41 5102,-41 5108,-41 5114,-47 5114,-53 5114,-53 5114,-104 5114,-104 5114,-110 5108,-116 5102,-116 5102,-116 4786,-116 4786,-116 4780,-116 4774,-110 4774,-104 4774,-104 4774,-53 4774,-53 4774,-47 4780,-41 4786,-41\"/>\n",
       "<text text-anchor=\"middle\" x=\"4944\" y=\"-100.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust22\" class=\"cluster\"><title>cluster_final</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M5236,-34C5236,-34 5653,-34 5653,-34 5659,-34 5665,-40 5665,-46 5665,-46 5665,-151 5665,-151 5665,-157 5659,-163 5653,-163 5653,-163 5236,-163 5236,-163 5230,-163 5224,-157 5224,-151 5224,-151 5224,-46 5224,-46 5224,-40 5230,-34 5236,-34\"/>\n",
       "<text text-anchor=\"middle\" x=\"5444.5\" y=\"-147.8\" font-family=\"Times,serif\" font-size=\"14.00\">final</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-96C132,-96 102,-96 102,-96 96,-96 90,-90 90,-84 90,-84 90,-72 90,-72 90,-66 96,-60 102,-60 102,-60 132,-60 132,-60 138,-60 144,-66 144,-72 144,-72 144,-84 144,-84 144,-90 138,-96 132,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1 -->\n",
       "<g id=\"node2\" class=\"node\"><title>layer1_block0_bn1</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-96C222,-96 192,-96 192,-96 186,-96 180,-90 180,-84 180,-84 180,-72 180,-72 180,-66 186,-60 192,-60 192,-60 222,-60 222,-60 228,-60 234,-66 234,-72 234,-72 234,-84 234,-84 234,-90 228,-96 222,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;layer1_block0_bn1 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;layer1_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-78C152.393,-78 161.311,-78 169.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-81.5001 179.919,-78 169.919,-74.5001 169.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>layer1_block0_relu1</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-96C312,-96 282,-96 282,-96 276,-96 270,-90 270,-84 270,-84 270,-72 270,-72 270,-66 276,-60 282,-60 282,-60 312,-60 312,-60 318,-60 324,-66 324,-72 324,-72 324,-84 324,-84 324,-90 318,-96 312,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1&#45;&gt;layer1_block0_relu1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>layer1_block0_bn1&#45;&gt;layer1_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-78C242.393,-78 251.311,-78 259.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-81.5001 269.919,-78 259.919,-74.5001 259.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_block0_branch_conv1</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-76C402,-76 372,-76 372,-76 366,-76 360,-70 360,-64 360,-64 360,-52 360,-52 360,-46 366,-40 372,-40 372,-40 402,-40 402,-40 408,-40 414,-46 414,-52 414,-52 414,-64 414,-64 414,-70 408,-76 402,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-71.9993C332.481,-70.1634 341.507,-68.1121 350.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"350.943,-69.5568 359.919,-63.9275 349.392,-62.7308 350.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_add -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer1_block0_add</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M762,-96C762,-96 732,-96 732,-96 726,-96 720,-90 720,-84 720,-84 720,-72 720,-72 720,-66 726,-60 732,-60 732,-60 762,-60 762,-60 768,-60 774,-66 774,-72 774,-72 774,-84 774,-84 774,-90 768,-96 762,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"747\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M321.737,-96.1018C330.735,-101.964 341.381,-107.78 352,-111 404.782,-127.006 420.844,-116 476,-116 476,-116 476,-116 568,-116 623.156,-116 639.218,-127.006 692,-111 699.301,-108.786 706.614,-105.345 713.396,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"715.545,-104.295 722.263,-96.1018 711.902,-98.3176 715.545,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2 -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_block0_branch_bn2</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-76C492,-76 462,-76 462,-76 456,-76 450,-70 450,-64 450,-64 450,-52 450,-52 450,-46 456,-40 462,-40 462,-40 492,-40 492,-40 498,-40 504,-46 504,-52 504,-52 504,-64 504,-64 504,-70 498,-76 492,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-58C422.393,-58 431.311,-58 439.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-61.5001 449.919,-58 439.919,-54.5001 439.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2 -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer1_block0_branch_relu2</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-76C582,-76 552,-76 552,-76 546,-76 540,-70 540,-64 540,-64 540,-52 540,-52 540,-46 546,-40 552,-40 552,-40 582,-40 582,-40 588,-40 594,-46 594,-52 594,-52 594,-64 594,-64 594,-70 588,-76 582,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-58C512.393,-58 521.311,-58 529.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-61.5001 539.919,-58 529.919,-54.5001 529.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2 -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer1_block0_branch_conv2</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M672,-76C672,-76 642,-76 642,-76 636,-76 630,-70 630,-64 630,-64 630,-52 630,-52 630,-46 636,-40 642,-40 642,-40 672,-40 672,-40 678,-40 684,-46 684,-52 684,-52 684,-64 684,-64 684,-70 678,-76 672,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-58C602.393,-58 611.311,-58 619.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-61.5001 629.919,-58 619.919,-54.5001 619.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer1_block0_branch_conv2&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.403,-64.0007C692.481,-65.8366 701.507,-67.8879 710.105,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"709.392,-73.2692 719.919,-72.0725 710.943,-66.4432 709.392,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1 -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer1_block1_bn1</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M852,-96C852,-96 822,-96 822,-96 816,-96 810,-90 810,-84 810,-84 810,-72 810,-72 810,-66 816,-60 822,-60 822,-60 852,-60 852,-60 858,-60 864,-66 864,-72 864,-72 864,-84 864,-84 864,-90 858,-96 852,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_add&#45;&gt;layer1_block1_bn1 -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer1_block0_add&#45;&gt;layer1_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.403,-78C782.393,-78 791.311,-78 799.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"799.919,-81.5001 809.919,-78 799.919,-74.5001 799.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1 -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer1_block1_relu1</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M942,-96C942,-96 912,-96 912,-96 906,-96 900,-90 900,-84 900,-84 900,-72 900,-72 900,-66 906,-60 912,-60 912,-60 942,-60 942,-60 948,-60 954,-66 954,-72 954,-72 954,-84 954,-84 954,-90 948,-96 942,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1&#45;&gt;layer1_block1_relu1 -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer1_block1_bn1&#45;&gt;layer1_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.403,-78C872.393,-78 881.311,-78 889.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"889.919,-81.5001 899.919,-78 889.919,-74.5001 889.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1 -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer1_block1_branch_conv1</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1032,-76C1032,-76 1002,-76 1002,-76 996,-76 990,-70 990,-64 990,-64 990,-52 990,-52 990,-46 996,-40 1002,-40 1002,-40 1032,-40 1032,-40 1038,-40 1044,-46 1044,-52 1044,-52 1044,-64 1044,-64 1044,-70 1038,-76 1032,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1 -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M954.403,-71.9993C962.481,-70.1634 971.507,-68.1121 980.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"980.943,-69.5568 989.919,-63.9275 979.392,-62.7308 980.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_add -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer1_block1_add</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1392,-96C1392,-96 1362,-96 1362,-96 1356,-96 1350,-90 1350,-84 1350,-84 1350,-72 1350,-72 1350,-66 1356,-60 1362,-60 1362,-60 1392,-60 1392,-60 1398,-60 1404,-66 1404,-72 1404,-72 1404,-84 1404,-84 1404,-90 1398,-96 1392,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1377\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M951.737,-96.1018C960.735,-101.964 971.381,-107.78 982,-111 1034.78,-127.006 1050.84,-116 1106,-116 1106,-116 1106,-116 1198,-116 1253.16,-116 1269.22,-127.006 1322,-111 1329.3,-108.786 1336.61,-105.345 1343.4,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1345.55,-104.295 1352.26,-96.1018 1341.9,-98.3176 1345.55,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2 -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer1_block1_branch_bn2</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1122,-76C1122,-76 1092,-76 1092,-76 1086,-76 1080,-70 1080,-64 1080,-64 1080,-52 1080,-52 1080,-46 1086,-40 1092,-40 1092,-40 1122,-40 1122,-40 1128,-40 1134,-46 1134,-52 1134,-52 1134,-64 1134,-64 1134,-70 1128,-76 1122,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2 -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.4,-58C1052.39,-58 1061.31,-58 1069.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1069.92,-61.5001 1079.92,-58 1069.92,-54.5001 1069.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2 -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer1_block1_branch_relu2</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1212,-76C1212,-76 1182,-76 1182,-76 1176,-76 1170,-70 1170,-64 1170,-64 1170,-52 1170,-52 1170,-46 1176,-40 1182,-40 1182,-40 1212,-40 1212,-40 1218,-40 1224,-46 1224,-52 1224,-52 1224,-64 1224,-64 1224,-70 1218,-76 1212,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2 -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1134.4,-58C1142.39,-58 1151.31,-58 1159.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1159.92,-61.5001 1169.92,-58 1159.92,-54.5001 1159.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2 -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer1_block1_branch_conv2</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1302,-76C1302,-76 1272,-76 1272,-76 1266,-76 1260,-70 1260,-64 1260,-64 1260,-52 1260,-52 1260,-46 1266,-40 1272,-40 1272,-40 1302,-40 1302,-40 1308,-40 1314,-46 1314,-52 1314,-52 1314,-64 1314,-64 1314,-70 1308,-76 1302,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1287\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2 -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1224.4,-58C1232.39,-58 1241.31,-58 1249.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1249.92,-61.5001 1259.92,-58 1249.92,-54.5001 1249.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>layer1_block1_branch_conv2&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1314.4,-64.0007C1322.48,-65.8366 1331.51,-67.8879 1340.1,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1339.39,-73.2692 1349.92,-72.0725 1340.94,-66.4432 1339.39,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1 -->\n",
       "<g id=\"node16\" class=\"node\"><title>layer2_block0_bn1</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1486,-96C1486,-96 1456,-96 1456,-96 1450,-96 1444,-90 1444,-84 1444,-84 1444,-72 1444,-72 1444,-66 1450,-60 1456,-60 1456,-60 1486,-60 1486,-60 1492,-60 1498,-66 1498,-72 1498,-72 1498,-84 1498,-84 1498,-90 1492,-96 1486,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1471\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_add&#45;&gt;layer2_block0_bn1 -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>layer1_block1_add&#45;&gt;layer2_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1404.12,-78C1413.34,-78 1423.89,-78 1433.8,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1433.82,-81.5001 1443.82,-78 1433.82,-74.5001 1433.82,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1 -->\n",
       "<g id=\"node17\" class=\"node\"><title>layer2_block0_relu1</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1576,-96C1576,-96 1546,-96 1546,-96 1540,-96 1534,-90 1534,-84 1534,-84 1534,-72 1534,-72 1534,-66 1540,-60 1546,-60 1546,-60 1576,-60 1576,-60 1582,-60 1588,-66 1588,-72 1588,-72 1588,-84 1588,-84 1588,-90 1582,-96 1576,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1561\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1&#45;&gt;layer2_block0_relu1 -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>layer2_block0_bn1&#45;&gt;layer2_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1498.4,-78C1506.39,-78 1515.31,-78 1523.82,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1523.92,-81.5001 1533.92,-78 1523.92,-74.5001 1523.92,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1 -->\n",
       "<g id=\"node18\" class=\"node\"><title>layer2_block0_branch_conv1</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1666,-77C1666,-77 1636,-77 1636,-77 1630,-77 1624,-71 1624,-65 1624,-65 1624,-53 1624,-53 1624,-47 1630,-41 1636,-41 1636,-41 1666,-41 1666,-41 1672,-41 1678,-47 1678,-53 1678,-53 1678,-65 1678,-65 1678,-71 1672,-77 1666,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1651\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1 -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1588.4,-72.2994C1596.48,-70.5552 1605.51,-68.6065 1614.1,-66.7501\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1614.88,-70.1628 1623.92,-64.6311 1613.41,-63.3205 1614.88,-70.1628\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3 -->\n",
       "<g id=\"node22\" class=\"node\"><title>layer2_block0_conv3</title>\n",
       "<g id=\"a_node22\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-152C1936,-152 1906,-152 1906,-152 1900,-152 1894,-146 1894,-140 1894,-140 1894,-128 1894,-128 1894,-122 1900,-116 1906,-116 1906,-116 1936,-116 1936,-116 1942,-116 1948,-122 1948,-128 1948,-128 1948,-140 1948,-140 1948,-146 1942,-152 1936,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-130.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_conv3 -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1585.19,-96.1046C1594.3,-102.243 1605.16,-108.442 1616,-112 1708.77,-142.443 1825.74,-140.431 1883.83,-136.933\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1884.22,-140.415 1893.97,-136.27 1883.76,-133.43 1884.22,-140.415\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2 -->\n",
       "<g id=\"node19\" class=\"node\"><title>layer2_block0_branch_bn2</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1756,-77C1756,-77 1726,-77 1726,-77 1720,-77 1714,-71 1714,-65 1714,-65 1714,-53 1714,-53 1714,-47 1720,-41 1726,-41 1726,-41 1756,-41 1756,-41 1762,-41 1768,-47 1768,-53 1768,-53 1768,-65 1768,-65 1768,-71 1762,-77 1756,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2 -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1678.4,-59C1686.39,-59 1695.31,-59 1703.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1703.92,-62.5001 1713.92,-59 1703.92,-55.5001 1703.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2 -->\n",
       "<g id=\"node20\" class=\"node\"><title>layer2_block0_branch_relu2</title>\n",
       "<g id=\"a_node20\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1846,-77C1846,-77 1816,-77 1816,-77 1810,-77 1804,-71 1804,-65 1804,-65 1804,-53 1804,-53 1804,-47 1810,-41 1816,-41 1816,-41 1846,-41 1846,-41 1852,-41 1858,-47 1858,-53 1858,-53 1858,-65 1858,-65 1858,-71 1852,-77 1846,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1831\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2 -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1768.4,-59C1776.39,-59 1785.31,-59 1793.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1793.92,-62.5001 1803.92,-59 1793.92,-55.5001 1793.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2 -->\n",
       "<g id=\"node21\" class=\"node\"><title>layer2_block0_branch_conv2</title>\n",
       "<g id=\"a_node21\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-77C1936,-77 1906,-77 1906,-77 1900,-77 1894,-71 1894,-65 1894,-65 1894,-53 1894,-53 1894,-47 1900,-41 1906,-41 1906,-41 1936,-41 1936,-41 1942,-41 1948,-47 1948,-53 1948,-53 1948,-65 1948,-65 1948,-71 1942,-77 1936,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2 -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1858.4,-59C1866.39,-59 1875.31,-59 1883.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1883.92,-62.5001 1893.92,-59 1883.92,-55.5001 1883.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_add -->\n",
       "<g id=\"node23\" class=\"node\"><title>layer2_block0_add</title>\n",
       "<g id=\"a_node23\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2026,-97C2026,-97 1996,-97 1996,-97 1990,-97 1984,-91 1984,-85 1984,-85 1984,-73 1984,-73 1984,-67 1990,-61 1996,-61 1996,-61 2026,-61 2026,-61 2032,-61 2038,-67 2038,-73 2038,-73 2038,-85 2038,-85 2038,-91 2032,-97 2026,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2011\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>layer2_block0_branch_conv2&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-65.0007C1956.48,-66.8366 1965.51,-68.8879 1974.1,-70.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1973.39,-74.2692 1983.92,-73.0725 1974.94,-67.4432 1973.39,-74.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>layer2_block0_conv3&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-117.498C1956.83,-112.23 1966.29,-106.317 1975.22,-100.735\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1977.29,-103.569 1983.92,-95.3006 1973.58,-97.6327 1977.29,-103.569\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1 -->\n",
       "<g id=\"node24\" class=\"node\"><title>layer2_block1_bn1</title>\n",
       "<g id=\"a_node24\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2116,-97C2116,-97 2086,-97 2086,-97 2080,-97 2074,-91 2074,-85 2074,-85 2074,-73 2074,-73 2074,-67 2080,-61 2086,-61 2086,-61 2116,-61 2116,-61 2122,-61 2128,-67 2128,-73 2128,-73 2128,-85 2128,-85 2128,-91 2122,-97 2116,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2101\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_add&#45;&gt;layer2_block1_bn1 -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>layer2_block0_add&#45;&gt;layer2_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2038.4,-79C2046.39,-79 2055.31,-79 2063.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2063.92,-82.5001 2073.92,-79 2063.92,-75.5001 2063.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1 -->\n",
       "<g id=\"node25\" class=\"node\"><title>layer2_block1_relu1</title>\n",
       "<g id=\"a_node25\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2206,-97C2206,-97 2176,-97 2176,-97 2170,-97 2164,-91 2164,-85 2164,-85 2164,-73 2164,-73 2164,-67 2170,-61 2176,-61 2176,-61 2206,-61 2206,-61 2212,-61 2218,-67 2218,-73 2218,-73 2218,-85 2218,-85 2218,-91 2212,-97 2206,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2191\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1&#45;&gt;layer2_block1_relu1 -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>layer2_block1_bn1&#45;&gt;layer2_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2128.4,-79C2136.39,-79 2145.31,-79 2153.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2153.92,-82.5001 2163.92,-79 2153.92,-75.5001 2153.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1 -->\n",
       "<g id=\"node26\" class=\"node\"><title>layer2_block1_branch_conv1</title>\n",
       "<g id=\"a_node26\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2296,-82C2296,-82 2266,-82 2266,-82 2260,-82 2254,-76 2254,-70 2254,-70 2254,-58 2254,-58 2254,-52 2260,-46 2266,-46 2266,-46 2296,-46 2296,-46 2302,-46 2308,-52 2308,-58 2308,-58 2308,-70 2308,-70 2308,-76 2302,-82 2296,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2281\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1 -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2218.4,-74.4995C2226.39,-73.1375 2235.31,-71.6175 2243.82,-70.1664\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2244.65,-73.5763 2253.92,-68.4456 2243.47,-66.6758 2244.65,-73.5763\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_add -->\n",
       "<g id=\"node30\" class=\"node\"><title>layer2_block1_add</title>\n",
       "<g id=\"a_node30\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2656,-82C2656,-82 2626,-82 2626,-82 2620,-82 2614,-76 2614,-70 2614,-70 2614,-58 2614,-58 2614,-52 2620,-46 2626,-46 2626,-46 2656,-46 2656,-46 2662,-46 2668,-52 2668,-58 2668,-58 2668,-70 2668,-70 2668,-76 2662,-82 2656,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2641\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge33\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2211.74,-97.0337C2221.35,-104.671 2233.51,-112.769 2246,-117 2298.24,-134.703 2314.84,-122 2370,-122 2370,-122 2370,-122 2462,-122 2517.16,-122 2535.38,-138.893 2586,-117 2599.55,-111.141 2611.6,-100.216 2620.87,-89.7996\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2623.65,-91.9369 2627.39,-82.0259 2618.28,-87.4385 2623.65,-91.9369\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2 -->\n",
       "<g id=\"node27\" class=\"node\"><title>layer2_block1_branch_bn2</title>\n",
       "<g id=\"a_node27\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2386,-82C2386,-82 2356,-82 2356,-82 2350,-82 2344,-76 2344,-70 2344,-70 2344,-58 2344,-58 2344,-52 2350,-46 2356,-46 2356,-46 2386,-46 2386,-46 2392,-46 2398,-52 2398,-58 2398,-58 2398,-70 2398,-70 2398,-76 2392,-82 2386,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2 -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2308.4,-64C2316.39,-64 2325.31,-64 2333.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2333.92,-67.5001 2343.92,-64 2333.92,-60.5001 2333.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2 -->\n",
       "<g id=\"node28\" class=\"node\"><title>layer2_block1_branch_relu2</title>\n",
       "<g id=\"a_node28\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2476,-82C2476,-82 2446,-82 2446,-82 2440,-82 2434,-76 2434,-70 2434,-70 2434,-58 2434,-58 2434,-52 2440,-46 2446,-46 2446,-46 2476,-46 2476,-46 2482,-46 2488,-52 2488,-58 2488,-58 2488,-70 2488,-70 2488,-76 2482,-82 2476,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2461\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2 -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2398.4,-64C2406.39,-64 2415.31,-64 2423.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2423.92,-67.5001 2433.92,-64 2423.92,-60.5001 2423.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2 -->\n",
       "<g id=\"node29\" class=\"node\"><title>layer2_block1_branch_conv2</title>\n",
       "<g id=\"a_node29\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2566,-82C2566,-82 2536,-82 2536,-82 2530,-82 2524,-76 2524,-70 2524,-70 2524,-58 2524,-58 2524,-52 2530,-46 2536,-46 2536,-46 2566,-46 2566,-46 2572,-46 2578,-52 2578,-58 2578,-58 2578,-70 2578,-70 2578,-76 2572,-82 2566,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2551\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2 -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2488.4,-64C2496.39,-64 2505.31,-64 2513.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2513.92,-67.5001 2523.92,-64 2513.92,-60.5001 2513.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge34\" class=\"edge\"><title>layer2_block1_branch_conv2&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2578.4,-64C2586.39,-64 2595.31,-64 2603.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2603.92,-67.5001 2613.92,-64 2603.92,-60.5001 2603.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1 -->\n",
       "<g id=\"node31\" class=\"node\"><title>layer3_block0_bn1</title>\n",
       "<g id=\"a_node31\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2750,-82C2750,-82 2720,-82 2720,-82 2714,-82 2708,-76 2708,-70 2708,-70 2708,-58 2708,-58 2708,-52 2714,-46 2720,-46 2720,-46 2750,-46 2750,-46 2756,-46 2762,-52 2762,-58 2762,-58 2762,-70 2762,-70 2762,-76 2756,-82 2750,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2735\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_add&#45;&gt;layer3_block0_bn1 -->\n",
       "<g id=\"edge35\" class=\"edge\"><title>layer2_block1_add&#45;&gt;layer3_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2668.12,-64C2677.34,-64 2687.89,-64 2697.8,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2697.82,-67.5001 2707.82,-64 2697.82,-60.5001 2697.82,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1 -->\n",
       "<g id=\"node32\" class=\"node\"><title>layer3_block0_relu1</title>\n",
       "<g id=\"a_node32\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2840,-82C2840,-82 2810,-82 2810,-82 2804,-82 2798,-76 2798,-70 2798,-70 2798,-58 2798,-58 2798,-52 2804,-46 2810,-46 2810,-46 2840,-46 2840,-46 2846,-46 2852,-52 2852,-58 2852,-58 2852,-70 2852,-70 2852,-76 2846,-82 2840,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2825\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1&#45;&gt;layer3_block0_relu1 -->\n",
       "<g id=\"edge36\" class=\"edge\"><title>layer3_block0_bn1&#45;&gt;layer3_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2762.4,-64C2770.39,-64 2779.31,-64 2787.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2787.92,-67.5001 2797.92,-64 2787.92,-60.5001 2787.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1 -->\n",
       "<g id=\"node33\" class=\"node\"><title>layer3_block0_branch_conv1</title>\n",
       "<g id=\"a_node33\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2930,-72C2930,-72 2900,-72 2900,-72 2894,-72 2888,-66 2888,-60 2888,-60 2888,-48 2888,-48 2888,-42 2894,-36 2900,-36 2900,-36 2930,-36 2930,-36 2936,-36 2942,-42 2942,-48 2942,-48 2942,-60 2942,-60 2942,-66 2936,-72 2930,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"2915\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1 -->\n",
       "<g id=\"edge37\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2852.4,-60.9997C2860.39,-60.0917 2869.31,-59.0783 2877.82,-58.1109\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2878.38,-61.5706 2887.92,-56.9638 2877.59,-54.6153 2878.38,-61.5706\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3 -->\n",
       "<g id=\"node37\" class=\"node\"><title>layer3_block0_conv3</title>\n",
       "<g id=\"a_node37\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-147C3200,-147 3170,-147 3170,-147 3164,-147 3158,-141 3158,-135 3158,-135 3158,-123 3158,-123 3158,-117 3164,-111 3170,-111 3170,-111 3200,-111 3200,-111 3206,-111 3212,-117 3212,-123 3212,-123 3212,-135 3212,-135 3212,-141 3206,-147 3200,-147\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-125.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_conv3 -->\n",
       "<g id=\"edge41\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2843.22,-82.1953C2853.06,-91.336 2866.16,-101.594 2880,-107 2970.95,-142.522 3088.72,-138.267 3147.4,-133.121\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3148.02,-136.578 3157.65,-132.159 3147.37,-129.609 3148.02,-136.578\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2 -->\n",
       "<g id=\"node34\" class=\"node\"><title>layer3_block0_branch_bn2</title>\n",
       "<g id=\"a_node34\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3020,-72C3020,-72 2990,-72 2990,-72 2984,-72 2978,-66 2978,-60 2978,-60 2978,-48 2978,-48 2978,-42 2984,-36 2990,-36 2990,-36 3020,-36 3020,-36 3026,-36 3032,-42 3032,-48 3032,-48 3032,-60 3032,-60 3032,-66 3026,-72 3020,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2 -->\n",
       "<g id=\"edge38\" class=\"edge\"><title>layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2942.4,-54C2950.39,-54 2959.31,-54 2967.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2967.92,-57.5001 2977.92,-54 2967.92,-50.5001 2967.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2 -->\n",
       "<g id=\"node35\" class=\"node\"><title>layer3_block0_branch_relu2</title>\n",
       "<g id=\"a_node35\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3110,-72C3110,-72 3080,-72 3080,-72 3074,-72 3068,-66 3068,-60 3068,-60 3068,-48 3068,-48 3068,-42 3074,-36 3080,-36 3080,-36 3110,-36 3110,-36 3116,-36 3122,-42 3122,-48 3122,-48 3122,-60 3122,-60 3122,-66 3116,-72 3110,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3095\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2 -->\n",
       "<g id=\"edge39\" class=\"edge\"><title>layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3032.4,-54C3040.39,-54 3049.31,-54 3057.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3057.92,-57.5001 3067.92,-54 3057.92,-50.5001 3057.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2 -->\n",
       "<g id=\"node36\" class=\"node\"><title>layer3_block0_branch_conv2</title>\n",
       "<g id=\"a_node36\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-72C3200,-72 3170,-72 3170,-72 3164,-72 3158,-66 3158,-60 3158,-60 3158,-48 3158,-48 3158,-42 3164,-36 3170,-36 3170,-36 3200,-36 3200,-36 3206,-36 3212,-42 3212,-48 3212,-48 3212,-60 3212,-60 3212,-66 3206,-72 3200,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2 -->\n",
       "<g id=\"edge40\" class=\"edge\"><title>layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3122.4,-54C3130.39,-54 3139.31,-54 3147.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3147.92,-57.5001 3157.92,-54 3147.92,-50.5001 3147.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_add -->\n",
       "<g id=\"node38\" class=\"node\"><title>layer3_block0_add</title>\n",
       "<g id=\"a_node38\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3290,-109C3290,-109 3260,-109 3260,-109 3254,-109 3248,-103 3248,-97 3248,-97 3248,-85 3248,-85 3248,-79 3254,-73 3260,-73 3260,-73 3290,-73 3290,-73 3296,-73 3302,-79 3302,-85 3302,-85 3302,-97 3302,-97 3302,-103 3296,-109 3290,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3275\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge43\" class=\"edge\"><title>layer3_block0_branch_conv2&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-65.1012C3220.66,-68.5716 3229.9,-72.4578 3238.66,-76.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3237.34,-79.3846 3247.92,-80.0341 3240.06,-72.9318 3237.34,-79.3846\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge42\" class=\"edge\"><title>layer3_block0_conv3&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-117.599C3220.66,-114.035 3229.9,-110.043 3238.66,-106.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3240.13,-109.44 3247.92,-102.262 3237.35,-103.013 3240.13,-109.44\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1 -->\n",
       "<g id=\"node39\" class=\"node\"><title>layer3_block1_bn1</title>\n",
       "<g id=\"a_node39\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3380,-109C3380,-109 3350,-109 3350,-109 3344,-109 3338,-103 3338,-97 3338,-97 3338,-85 3338,-85 3338,-79 3344,-73 3350,-73 3350,-73 3380,-73 3380,-73 3386,-73 3392,-79 3392,-85 3392,-85 3392,-97 3392,-97 3392,-103 3386,-109 3380,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3365\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_add&#45;&gt;layer3_block1_bn1 -->\n",
       "<g id=\"edge44\" class=\"edge\"><title>layer3_block0_add&#45;&gt;layer3_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3302.4,-91C3310.39,-91 3319.31,-91 3327.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3327.92,-94.5001 3337.92,-91 3327.92,-87.5001 3327.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1 -->\n",
       "<g id=\"node40\" class=\"node\"><title>layer3_block1_relu1</title>\n",
       "<g id=\"a_node40\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3470,-109C3470,-109 3440,-109 3440,-109 3434,-109 3428,-103 3428,-97 3428,-97 3428,-85 3428,-85 3428,-79 3434,-73 3440,-73 3440,-73 3470,-73 3470,-73 3476,-73 3482,-79 3482,-85 3482,-85 3482,-97 3482,-97 3482,-103 3476,-109 3470,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3455\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1&#45;&gt;layer3_block1_relu1 -->\n",
       "<g id=\"edge45\" class=\"edge\"><title>layer3_block1_bn1&#45;&gt;layer3_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3392.4,-91C3400.39,-91 3409.31,-91 3417.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3417.92,-94.5001 3427.92,-91 3417.92,-87.5001 3417.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1 -->\n",
       "<g id=\"node41\" class=\"node\"><title>layer3_block1_branch_conv1</title>\n",
       "<g id=\"a_node41\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3560,-89C3560,-89 3530,-89 3530,-89 3524,-89 3518,-83 3518,-77 3518,-77 3518,-65 3518,-65 3518,-59 3524,-53 3530,-53 3530,-53 3560,-53 3560,-53 3566,-53 3572,-59 3572,-65 3572,-65 3572,-77 3572,-77 3572,-83 3566,-89 3560,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3545\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1 -->\n",
       "<g id=\"edge46\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3482.4,-84.9993C3490.48,-83.1634 3499.51,-81.1121 3508.1,-79.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3508.94,-82.5568 3517.92,-76.9275 3507.39,-75.7308 3508.94,-82.5568\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_add -->\n",
       "<g id=\"node45\" class=\"node\"><title>layer3_block1_add</title>\n",
       "<g id=\"a_node45\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3920,-109C3920,-109 3890,-109 3890,-109 3884,-109 3878,-103 3878,-97 3878,-97 3878,-85 3878,-85 3878,-79 3884,-73 3890,-73 3890,-73 3920,-73 3920,-73 3926,-73 3932,-79 3932,-85 3932,-85 3932,-97 3932,-97 3932,-103 3926,-109 3920,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3905\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge50\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3479.74,-109.102C3488.74,-114.964 3499.38,-120.78 3510,-124 3562.78,-140.006 3578.84,-129 3634,-129 3634,-129 3634,-129 3726,-129 3781.16,-129 3797.22,-140.006 3850,-124 3857.3,-121.786 3864.61,-118.345 3871.4,-114.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3873.55,-117.295 3880.26,-109.102 3869.9,-111.318 3873.55,-117.295\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2 -->\n",
       "<g id=\"node42\" class=\"node\"><title>layer3_block1_branch_bn2</title>\n",
       "<g id=\"a_node42\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3650,-89C3650,-89 3620,-89 3620,-89 3614,-89 3608,-83 3608,-77 3608,-77 3608,-65 3608,-65 3608,-59 3614,-53 3620,-53 3620,-53 3650,-53 3650,-53 3656,-53 3662,-59 3662,-65 3662,-65 3662,-77 3662,-77 3662,-83 3656,-89 3650,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2 -->\n",
       "<g id=\"edge47\" class=\"edge\"><title>layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3572.4,-71C3580.39,-71 3589.31,-71 3597.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3597.92,-74.5001 3607.92,-71 3597.92,-67.5001 3597.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2 -->\n",
       "<g id=\"node43\" class=\"node\"><title>layer3_block1_branch_relu2</title>\n",
       "<g id=\"a_node43\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3740,-89C3740,-89 3710,-89 3710,-89 3704,-89 3698,-83 3698,-77 3698,-77 3698,-65 3698,-65 3698,-59 3704,-53 3710,-53 3710,-53 3740,-53 3740,-53 3746,-53 3752,-59 3752,-65 3752,-65 3752,-77 3752,-77 3752,-83 3746,-89 3740,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3725\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2 -->\n",
       "<g id=\"edge48\" class=\"edge\"><title>layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3662.4,-71C3670.39,-71 3679.31,-71 3687.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3687.92,-74.5001 3697.92,-71 3687.92,-67.5001 3687.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2 -->\n",
       "<g id=\"node44\" class=\"node\"><title>layer3_block1_branch_conv2</title>\n",
       "<g id=\"a_node44\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3830,-89C3830,-89 3800,-89 3800,-89 3794,-89 3788,-83 3788,-77 3788,-77 3788,-65 3788,-65 3788,-59 3794,-53 3800,-53 3800,-53 3830,-53 3830,-53 3836,-53 3842,-59 3842,-65 3842,-65 3842,-77 3842,-77 3842,-83 3836,-89 3830,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3815\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2 -->\n",
       "<g id=\"edge49\" class=\"edge\"><title>layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3752.4,-71C3760.39,-71 3769.31,-71 3777.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3777.92,-74.5001 3787.92,-71 3777.92,-67.5001 3777.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge51\" class=\"edge\"><title>layer3_block1_branch_conv2&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3842.4,-77.0007C3850.48,-78.8366 3859.51,-80.8879 3868.1,-82.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3867.39,-86.2692 3877.92,-85.0725 3868.94,-79.4432 3867.39,-86.2692\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1 -->\n",
       "<g id=\"node46\" class=\"node\"><title>layer4_block0_bn1</title>\n",
       "<g id=\"a_node46\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4014,-109C4014,-109 3984,-109 3984,-109 3978,-109 3972,-103 3972,-97 3972,-97 3972,-85 3972,-85 3972,-79 3978,-73 3984,-73 3984,-73 4014,-73 4014,-73 4020,-73 4026,-79 4026,-85 4026,-85 4026,-97 4026,-97 4026,-103 4020,-109 4014,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3999\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_add&#45;&gt;layer4_block0_bn1 -->\n",
       "<g id=\"edge52\" class=\"edge\"><title>layer3_block1_add&#45;&gt;layer4_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3932.12,-91C3941.34,-91 3951.89,-91 3961.8,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3961.82,-94.5001 3971.82,-91 3961.82,-87.5001 3961.82,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1 -->\n",
       "<g id=\"node47\" class=\"node\"><title>layer4_block0_relu1</title>\n",
       "<g id=\"a_node47\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4104,-109C4104,-109 4074,-109 4074,-109 4068,-109 4062,-103 4062,-97 4062,-97 4062,-85 4062,-85 4062,-79 4068,-73 4074,-73 4074,-73 4104,-73 4104,-73 4110,-73 4116,-79 4116,-85 4116,-85 4116,-97 4116,-97 4116,-103 4110,-109 4104,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"4089\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1&#45;&gt;layer4_block0_relu1 -->\n",
       "<g id=\"edge53\" class=\"edge\"><title>layer4_block0_bn1&#45;&gt;layer4_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4026.4,-91C4034.39,-91 4043.31,-91 4051.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4051.92,-94.5001 4061.92,-91 4051.92,-87.5001 4051.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1 -->\n",
       "<g id=\"node48\" class=\"node\"><title>layer4_block0_branch_conv1</title>\n",
       "<g id=\"a_node48\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4194,-68C4194,-68 4164,-68 4164,-68 4158,-68 4152,-62 4152,-56 4152,-56 4152,-44 4152,-44 4152,-38 4158,-32 4164,-32 4164,-32 4194,-32 4194,-32 4200,-32 4206,-38 4206,-44 4206,-44 4206,-56 4206,-56 4206,-62 4200,-68 4194,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4179\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1 -->\n",
       "<g id=\"edge54\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.4,-78.6986C4124.66,-74.8531 4133.9,-70.5468 4142.66,-66.463\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4144.33,-69.5472 4151.92,-62.1514 4141.38,-63.2021 4144.33,-69.5472\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3 -->\n",
       "<g id=\"node52\" class=\"node\"><title>layer4_block0_conv3</title>\n",
       "<g id=\"a_node52\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-143C4464,-143 4434,-143 4434,-143 4428,-143 4422,-137 4422,-131 4422,-131 4422,-119 4422,-119 4422,-113 4428,-107 4434,-107 4434,-107 4464,-107 4464,-107 4470,-107 4476,-113 4476,-119 4476,-119 4476,-131 4476,-131 4476,-137 4470,-143 4464,-143\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-121.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_conv3 -->\n",
       "<g id=\"edge58\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.22,-97.6371C4125,-99.6421 4134.88,-101.664 4144,-103 4239.91,-117.051 4354.23,-122.258 4411.49,-124.083\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.73,-127.592 4421.83,-124.395 4411.94,-120.595 4411.73,-127.592\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2 -->\n",
       "<g id=\"node49\" class=\"node\"><title>layer4_block0_branch_bn2</title>\n",
       "<g id=\"a_node49\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4284,-68C4284,-68 4254,-68 4254,-68 4248,-68 4242,-62 4242,-56 4242,-56 4242,-44 4242,-44 4242,-38 4248,-32 4254,-32 4254,-32 4284,-32 4284,-32 4290,-32 4296,-38 4296,-44 4296,-44 4296,-56 4296,-56 4296,-62 4290,-68 4284,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2 -->\n",
       "<g id=\"edge55\" class=\"edge\"><title>layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4206.4,-50C4214.39,-50 4223.31,-50 4231.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4231.92,-53.5001 4241.92,-50 4231.92,-46.5001 4231.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2 -->\n",
       "<g id=\"node50\" class=\"node\"><title>layer4_block0_branch_relu2</title>\n",
       "<g id=\"a_node50\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4374,-68C4374,-68 4344,-68 4344,-68 4338,-68 4332,-62 4332,-56 4332,-56 4332,-44 4332,-44 4332,-38 4338,-32 4344,-32 4344,-32 4374,-32 4374,-32 4380,-32 4386,-38 4386,-44 4386,-44 4386,-56 4386,-56 4386,-62 4380,-68 4374,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4359\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2 -->\n",
       "<g id=\"edge56\" class=\"edge\"><title>layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4296.4,-50C4304.39,-50 4313.31,-50 4321.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4321.92,-53.5001 4331.92,-50 4321.92,-46.5001 4321.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2 -->\n",
       "<g id=\"node51\" class=\"node\"><title>layer4_block0_branch_conv2</title>\n",
       "<g id=\"a_node51\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-68C4464,-68 4434,-68 4434,-68 4428,-68 4422,-62 4422,-56 4422,-56 4422,-44 4422,-44 4422,-38 4428,-32 4434,-32 4434,-32 4464,-32 4464,-32 4470,-32 4476,-38 4476,-44 4476,-44 4476,-56 4476,-56 4476,-62 4470,-68 4464,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2 -->\n",
       "<g id=\"edge57\" class=\"edge\"><title>layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4386.4,-50C4394.39,-50 4403.31,-50 4411.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.92,-53.5001 4421.92,-50 4411.92,-46.5001 4411.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_add -->\n",
       "<g id=\"node53\" class=\"node\"><title>layer4_block0_add</title>\n",
       "<g id=\"a_node53\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M4554,-105C4554,-105 4524,-105 4524,-105 4518,-105 4512,-99 4512,-93 4512,-93 4512,-81 4512,-81 4512,-75 4518,-69 4524,-69 4524,-69 4554,-69 4554,-69 4560,-69 4566,-75 4566,-81 4566,-81 4566,-93 4566,-93 4566,-99 4560,-105 4554,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4539\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge60\" class=\"edge\"><title>layer4_block0_branch_conv2&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-61.1012C4484.66,-64.5716 4493.9,-68.4578 4502.66,-72.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4501.34,-75.3846 4511.92,-76.0341 4504.06,-68.9318 4501.34,-75.3846\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge59\" class=\"edge\"><title>layer4_block0_conv3&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-113.599C4484.66,-110.035 4493.9,-106.043 4502.66,-102.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4504.13,-105.44 4511.92,-98.2623 4501.35,-99.0135 4504.13,-105.44\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1 -->\n",
       "<g id=\"node54\" class=\"node\"><title>layer4_block1_bn1</title>\n",
       "<g id=\"a_node54\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4644,-105C4644,-105 4614,-105 4614,-105 4608,-105 4602,-99 4602,-93 4602,-93 4602,-81 4602,-81 4602,-75 4608,-69 4614,-69 4614,-69 4644,-69 4644,-69 4650,-69 4656,-75 4656,-81 4656,-81 4656,-93 4656,-93 4656,-99 4650,-105 4644,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4629\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_add&#45;&gt;layer4_block1_bn1 -->\n",
       "<g id=\"edge61\" class=\"edge\"><title>layer4_block0_add&#45;&gt;layer4_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4566.4,-87C4574.39,-87 4583.31,-87 4591.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4591.92,-90.5001 4601.92,-87 4591.92,-83.5001 4591.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1 -->\n",
       "<g id=\"node55\" class=\"node\"><title>layer4_block1_relu1</title>\n",
       "<g id=\"a_node55\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4734,-105C4734,-105 4704,-105 4704,-105 4698,-105 4692,-99 4692,-93 4692,-93 4692,-81 4692,-81 4692,-75 4698,-69 4704,-69 4704,-69 4734,-69 4734,-69 4740,-69 4746,-75 4746,-81 4746,-81 4746,-93 4746,-93 4746,-99 4740,-105 4734,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4719\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1&#45;&gt;layer4_block1_relu1 -->\n",
       "<g id=\"edge62\" class=\"edge\"><title>layer4_block1_bn1&#45;&gt;layer4_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4656.4,-87C4664.39,-87 4673.31,-87 4681.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4681.92,-90.5001 4691.92,-87 4681.92,-83.5001 4681.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1 -->\n",
       "<g id=\"node56\" class=\"node\"><title>layer4_block1_branch_conv1</title>\n",
       "<g id=\"a_node56\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4824,-85C4824,-85 4794,-85 4794,-85 4788,-85 4782,-79 4782,-73 4782,-73 4782,-61 4782,-61 4782,-55 4788,-49 4794,-49 4794,-49 4824,-49 4824,-49 4830,-49 4836,-55 4836,-61 4836,-61 4836,-73 4836,-73 4836,-79 4830,-85 4824,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4809\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1 -->\n",
       "<g id=\"edge63\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4746.4,-80.9993C4754.48,-79.1634 4763.51,-77.1121 4772.1,-75.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4772.94,-78.5568 4781.92,-72.9275 4771.39,-71.7308 4772.94,-78.5568\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_add -->\n",
       "<g id=\"node60\" class=\"node\"><title>layer4_block1_add</title>\n",
       "<g id=\"a_node60\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M5184,-105C5184,-105 5154,-105 5154,-105 5148,-105 5142,-99 5142,-93 5142,-93 5142,-81 5142,-81 5142,-75 5148,-69 5154,-69 5154,-69 5184,-69 5184,-69 5190,-69 5196,-75 5196,-81 5196,-81 5196,-93 5196,-93 5196,-99 5190,-105 5184,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5169\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge67\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4743.74,-105.102C4752.74,-110.964 4763.38,-116.78 4774,-120 4826.78,-136.006 4842.84,-125 4898,-125 4898,-125 4898,-125 4990,-125 5045.16,-125 5061.22,-136.006 5114,-120 5121.3,-117.786 5128.61,-114.345 5135.4,-110.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5137.55,-113.295 5144.26,-105.102 5133.9,-107.318 5137.55,-113.295\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2 -->\n",
       "<g id=\"node57\" class=\"node\"><title>layer4_block1_branch_bn2</title>\n",
       "<g id=\"a_node57\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4914,-85C4914,-85 4884,-85 4884,-85 4878,-85 4872,-79 4872,-73 4872,-73 4872,-61 4872,-61 4872,-55 4878,-49 4884,-49 4884,-49 4914,-49 4914,-49 4920,-49 4926,-55 4926,-61 4926,-61 4926,-73 4926,-73 4926,-79 4920,-85 4914,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2 -->\n",
       "<g id=\"edge64\" class=\"edge\"><title>layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4836.4,-67C4844.39,-67 4853.31,-67 4861.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4861.92,-70.5001 4871.92,-67 4861.92,-63.5001 4861.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2 -->\n",
       "<g id=\"node58\" class=\"node\"><title>layer4_block1_branch_relu2</title>\n",
       "<g id=\"a_node58\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M5004,-85C5004,-85 4974,-85 4974,-85 4968,-85 4962,-79 4962,-73 4962,-73 4962,-61 4962,-61 4962,-55 4968,-49 4974,-49 4974,-49 5004,-49 5004,-49 5010,-49 5016,-55 5016,-61 5016,-61 5016,-73 5016,-73 5016,-79 5010,-85 5004,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4989\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2 -->\n",
       "<g id=\"edge65\" class=\"edge\"><title>layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4926.4,-67C4934.39,-67 4943.31,-67 4951.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4951.92,-70.5001 4961.92,-67 4951.92,-63.5001 4951.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2 -->\n",
       "<g id=\"node59\" class=\"node\"><title>layer4_block1_branch_conv2</title>\n",
       "<g id=\"a_node59\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M5094,-85C5094,-85 5064,-85 5064,-85 5058,-85 5052,-79 5052,-73 5052,-73 5052,-61 5052,-61 5052,-55 5058,-49 5064,-49 5064,-49 5094,-49 5094,-49 5100,-49 5106,-55 5106,-61 5106,-61 5106,-73 5106,-73 5106,-79 5100,-85 5094,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"5079\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2 -->\n",
       "<g id=\"edge66\" class=\"edge\"><title>layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5016.4,-67C5024.39,-67 5033.31,-67 5041.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5041.92,-70.5001 5051.92,-67 5041.92,-63.5001 5041.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge68\" class=\"edge\"><title>layer4_block1_branch_conv2&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5106.4,-73.0007C5114.48,-74.8366 5123.51,-76.8879 5132.1,-78.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5131.39,-82.2692 5141.92,-81.0725 5132.94,-75.4432 5131.39,-82.2692\"/>\n",
       "</g>\n",
       "<!-- final_in -->\n",
       "<g id=\"node61\" class=\"node\"><title>final_in</title>\n",
       "<g id=\"a_node61\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5274,-105C5274,-105 5244,-105 5244,-105 5238,-105 5232,-99 5232,-93 5232,-93 5232,-81 5232,-81 5232,-75 5238,-69 5244,-69 5244,-69 5274,-69 5274,-69 5280,-69 5286,-75 5286,-81 5286,-81 5286,-93 5286,-93 5286,-99 5280,-105 5274,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5259\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">in</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_add&#45;&gt;final_in -->\n",
       "<g id=\"edge69\" class=\"edge\"><title>layer4_block1_add&#45;&gt;final_in</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5196.4,-87C5204.39,-87 5213.31,-87 5221.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5221.92,-90.5001 5231.92,-87 5221.92,-83.5001 5221.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_maxpool -->\n",
       "<g id=\"node62\" class=\"node\"><title>final_maxpool</title>\n",
       "<g id=\"a_node62\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#fdb462\" stroke=\"black\" d=\"M5375,-78C5375,-78 5334,-78 5334,-78 5328,-78 5322,-72 5322,-66 5322,-66 5322,-54 5322,-54 5322,-48 5328,-42 5334,-42 5334,-42 5375,-42 5375,-42 5381,-42 5387,-48 5387,-54 5387,-54 5387,-66 5387,-66 5387,-72 5381,-78 5375,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">maxpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_maxpool -->\n",
       "<g id=\"edge70\" class=\"edge\"><title>final_in&#45;&gt;final_maxpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-79.479C5294.13,-77.1436 5303.26,-74.509 5312.12,-71.9497\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.31,-75.2492 5321.94,-69.1121 5311.37,-68.524 5313.31,-75.2492\"/>\n",
       "</g>\n",
       "<!-- final_avgpool -->\n",
       "<g id=\"node63\" class=\"node\"><title>final_avgpool</title>\n",
       "<g id=\"a_node63\"><a xlink:title=\"AvgPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;ceil_mode&#39;: False, &#39;count_include_pad&#39;: True}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M5372.5,-132C5372.5,-132 5336.5,-132 5336.5,-132 5330.5,-132 5324.5,-126 5324.5,-120 5324.5,-120 5324.5,-108 5324.5,-108 5324.5,-102 5330.5,-96 5336.5,-96 5336.5,-96 5372.5,-96 5372.5,-96 5378.5,-96 5384.5,-102 5384.5,-108 5384.5,-108 5384.5,-120 5384.5,-120 5384.5,-126 5378.5,-132 5372.5,-132\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-110.3\" font-family=\"Times,serif\" font-size=\"14.00\">avgpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_avgpool -->\n",
       "<g id=\"edge71\" class=\"edge\"><title>final_in&#45;&gt;final_avgpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-94.521C5294.87,-97.0688 5304.92,-99.9726 5314.53,-102.746\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.73,-106.157 5324.3,-105.569 5315.67,-99.4322 5313.73,-106.157\"/>\n",
       "</g>\n",
       "<!-- final_concat -->\n",
       "<g id=\"node64\" class=\"node\"><title>final_concat</title>\n",
       "<g id=\"a_node64\"><a xlink:title=\"Concat {}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M5465,-105C5465,-105 5435,-105 5435,-105 5429,-105 5423,-99 5423,-93 5423,-93 5423,-81 5423,-81 5423,-75 5429,-69 5435,-69 5435,-69 5465,-69 5465,-69 5471,-69 5477,-75 5477,-81 5477,-81 5477,-93 5477,-93 5477,-99 5471,-105 5465,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5450\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">concat</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_maxpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge72\" class=\"edge\"><title>final_maxpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5387.15,-69.141C5395.53,-71.5601 5404.63,-74.1884 5413.21,-76.6659\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5412.39,-80.0717 5422.97,-79.4835 5414.33,-73.3465 5412.39,-80.0717\"/>\n",
       "</g>\n",
       "<!-- final_avgpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge73\" class=\"edge\"><title>final_avgpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5384.57,-105.605C5393.55,-103.013 5403.54,-100.127 5412.93,-97.4155\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5414.17,-100.702 5422.8,-94.5648 5412.22,-93.9766 5414.17,-100.702\"/>\n",
       "</g>\n",
       "<!-- final_flatten -->\n",
       "<g id=\"node65\" class=\"node\"><title>final_flatten</title>\n",
       "<g id=\"a_node65\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M5555,-105C5555,-105 5525,-105 5525,-105 5519,-105 5513,-99 5513,-93 5513,-93 5513,-81 5513,-81 5513,-75 5519,-69 5525,-69 5525,-69 5555,-69 5555,-69 5561,-69 5567,-75 5567,-81 5567,-81 5567,-93 5567,-93 5567,-99 5561,-105 5555,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5540\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_concat&#45;&gt;final_flatten -->\n",
       "<g id=\"edge74\" class=\"edge\"><title>final_concat&#45;&gt;final_flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5477.4,-87C5485.39,-87 5494.31,-87 5502.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5502.92,-90.5001 5512.92,-87 5502.92,-83.5001 5502.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_linear -->\n",
       "<g id=\"node66\" class=\"node\"><title>final_linear</title>\n",
       "<g id=\"a_node66\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: Parameter containing:\\ntensor([ 0.0226, &#45;0.0019, &#160;0.0280, &#160;0.0205, &#160;0.0199, &#45;0.0263, &#160;0.0408, &#160;0.0185,\\n &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#45;0.0401, &#160;0.0091], requires_grad=True)}\">\n",
       "<path fill=\"#ccebc5\" stroke=\"black\" d=\"M5645,-105C5645,-105 5615,-105 5615,-105 5609,-105 5603,-99 5603,-93 5603,-93 5603,-81 5603,-81 5603,-75 5609,-69 5615,-69 5615,-69 5645,-69 5645,-69 5651,-69 5657,-75 5657,-81 5657,-81 5657,-93 5657,-93 5657,-99 5651,-105 5645,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5630\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_flatten&#45;&gt;final_linear -->\n",
       "<g id=\"edge75\" class=\"edge\"><title>final_flatten&#45;&gt;final_linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5567.4,-87C5575.39,-87 5584.31,-87 5592.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5592.92,-90.5001 5602.92,-87 5592.92,-83.5001 5592.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node67\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node67\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5746,-105C5746,-105 5705,-105 5705,-105 5699,-105 5693,-99 5693,-93 5693,-93 5693,-81 5693,-81 5693,-75 5699,-69 5705,-69 5705,-69 5746,-69 5746,-69 5752,-69 5758,-75 5758,-81 5758,-81 5758,-93 5758,-93 5758,-99 5752,-105 5746,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5725.5\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_linear&#45;&gt;classifier -->\n",
       "<g id=\"edge76\" class=\"edge\"><title>final_linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5657.04,-87C5665.05,-87 5674.06,-87 5682.83,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5682.95,-90.5001 5692.94,-87 5682.94,-83.5001 5682.95,-90.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node68\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-96C42,-96 12,-96 12,-96 6,-96 0,-90 0,-84 0,-84 0,-72 0,-72 0,-66 6,-60 12,-60 12,-60 42,-60 42,-60 48,-60 54,-66 54,-72 54,-72 54,-84 54,-84 54,-90 48,-96 42,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-78C62.3932,-78 71.3106,-78 79.8241,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-81.5001 89.919,-78 79.919,-74.5001 79.919,-81.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7f0103ee1b38>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.0500       4.4296       1.7153       0.3614       0.3681       1.2575       0.5486       4.7976\n",
      "           2       0.1000       4.4320       1.1202       0.5958       0.3055       1.8857       0.4832       9.5352\n",
      "           3       0.1500       4.4300       0.8650       0.6948       0.3056       0.9143       0.6849      14.2708\n",
      "           4       0.2000       4.4295       0.7227       0.7451       0.3050       0.6717       0.7688      19.0053\n",
      "           5       0.2500       4.4632       0.6316       0.7794       0.3059       0.7078       0.7486      23.7744\n",
      "           6       0.3000       4.4235       0.5784       0.8000       0.3058       0.8209       0.7257      28.5037\n",
      "           7       0.3500       4.4219       0.5486       0.8085       0.3052       0.6884       0.7639      33.2309\n",
      "           8       0.4000       4.4210       0.5306       0.8156       0.3044       0.6368       0.7850      37.9563\n",
      "           9       0.3818       4.4554       0.5079       0.8252       0.3047       0.6339       0.7900      42.7164\n",
      "          10       0.3636       4.4135       0.4704       0.8357       0.3043       0.6429       0.7858      47.4342\n",
      "          11       0.3455       4.4166       0.4511       0.8453       0.3050       0.7037       0.7631      52.1558\n",
      "          12       0.3273       4.4083       0.4230       0.8539       0.3052       0.5583       0.8104      56.8693\n",
      "          13       0.3091       4.4474       0.4089       0.8590       0.3053       0.5325       0.8258      61.6219\n",
      "          14       0.2909       4.4161       0.3904       0.8659       0.3044       0.4932       0.8338      66.3424\n",
      "          15       0.2727       4.4134       0.3709       0.8713       0.3045       0.4594       0.8465      71.0603\n",
      "          16       0.2545       4.4139       0.3545       0.8770       0.3055       0.5219       0.8258      75.7796\n",
      "          17       0.2364       4.4452       0.3359       0.8827       0.3052       0.4402       0.8511      80.5301\n",
      "          18       0.2182       4.4099       0.3228       0.8877       0.3051       0.4664       0.8472      85.2451\n",
      "          19       0.2000       4.4116       0.3057       0.8941       0.3037       0.3760       0.8750      89.9605\n",
      "          20       0.1818       4.4101       0.2881       0.8993       0.3055       0.3868       0.8671      94.6761\n",
      "          21       0.1636       4.4452       0.2782       0.9043       0.3054       0.3691       0.8759      99.4267\n",
      "          22       0.1455       4.4108       0.2531       0.9126       0.3046       0.3198       0.8934     104.1421\n",
      "          23       0.1273       4.4109       0.2310       0.9192       0.3048       0.4297       0.8610     108.8578\n",
      "          24       0.1091       4.4097       0.2176       0.9237       0.3045       0.3234       0.8898     113.5720\n",
      "          25       0.0909       4.4468       0.1931       0.9320       0.3050       0.3360       0.8902     118.3239\n",
      "          26       0.0727       4.4120       0.1685       0.9407       0.3032       0.2577       0.9157     123.0391\n",
      "          27       0.0545       4.4092       0.1485       0.9476       0.3071       0.2486       0.9163     127.7554\n",
      "          28       0.0364       4.4116       0.1210       0.9588       0.3049       0.2228       0.9294     132.4718\n",
      "          29       0.0182       4.4306       0.0995       0.9665       0.3042       0.1987       0.9360     137.2066\n",
      "          30       0.0000       4.4111       0.0788       0.9734       0.3049       0.1862       0.9414     141.9226\n"
     ]
    }
   ],
   "source": [
    "lr_schedule = PiecewiseLinear([0, 8, 30], [0, 0.4, 0])\n",
    "batch_size = 512\n",
    "\n",
    "n = DAWN_net()\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(8,8)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 3: Regularisation](https://www.myrtle.ai/2018/09/24/how_to_train_your_resnet_3/)  - batch_size=768 (154s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"43pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 43.08\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.187305 0.187305) rotate(0) translate(4 226)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-226 5762,-226 5762,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-52C94,-52 140,-52 140,-52 146,-52 152,-58 152,-64 152,-64 152,-115 152,-115 152,-121 146,-127 140,-127 140,-127 94,-127 94,-127 88,-127 82,-121 82,-115 82,-115 82,-64 82,-64 82,-58 88,-52 94,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M176,-16C176,-16 1408,-16 1408,-16 1414,-16 1420,-22 1420,-28 1420,-28 1420,-167 1420,-167 1420,-173 1414,-179 1408,-179 1408,-179 176,-179 176,-179 170,-179 164,-173 164,-167 164,-167 164,-28 164,-28 164,-22 170,-16 176,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"792\" y=\"-163.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer1_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M184,-24C184,-24 770,-24 770,-24 776,-24 782,-30 782,-36 782,-36 782,-136 782,-136 782,-142 776,-148 770,-148 770,-148 184,-148 184,-148 178,-148 172,-142 172,-136 172,-136 172,-36 172,-36 172,-30 178,-24 184,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer1_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-32C364,-32 680,-32 680,-32 686,-32 692,-38 692,-44 692,-44 692,-95 692,-95 692,-101 686,-107 680,-107 680,-107 364,-107 364,-107 358,-107 352,-101 352,-95 352,-95 352,-44 352,-44 352,-38 358,-32 364,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer1_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M814,-24C814,-24 1400,-24 1400,-24 1406,-24 1412,-30 1412,-36 1412,-36 1412,-136 1412,-136 1412,-142 1406,-148 1400,-148 1400,-148 814,-148 814,-148 808,-148 802,-142 802,-136 802,-136 802,-36 802,-36 802,-30 808,-24 814,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-132.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer1_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M994,-32C994,-32 1310,-32 1310,-32 1316,-32 1322,-38 1322,-44 1322,-44 1322,-95 1322,-95 1322,-101 1316,-107 1310,-107 1310,-107 994,-107 994,-107 988,-107 982,-101 982,-95 982,-95 982,-44 982,-44 982,-38 988,-32 994,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"1152\" y=\"-91.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1440,-17C1440,-17 2672,-17 2672,-17 2678,-17 2684,-23 2684,-29 2684,-29 2684,-202 2684,-202 2684,-208 2678,-214 2672,-214 2672,-214 1440,-214 1440,-214 1434,-214 1428,-208 1428,-202 1428,-202 1428,-29 1428,-29 1428,-23 1434,-17 1440,-17\"/>\n",
       "<text text-anchor=\"middle\" x=\"2056\" y=\"-198.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_layer2_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1448,-25C1448,-25 2034,-25 2034,-25 2040,-25 2046,-31 2046,-37 2046,-37 2046,-171 2046,-171 2046,-177 2040,-183 2034,-183 2034,-183 1448,-183 1448,-183 1442,-183 1436,-177 1436,-171 1436,-171 1436,-37 1436,-37 1436,-31 1442,-25 1448,-25\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-167.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_layer2_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1628,-33C1628,-33 1944,-33 1944,-33 1950,-33 1956,-39 1956,-45 1956,-45 1956,-96 1956,-96 1956,-102 1950,-108 1944,-108 1944,-108 1628,-108 1628,-108 1622,-108 1616,-102 1616,-96 1616,-96 1616,-45 1616,-45 1616,-39 1622,-33 1628,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"1786\" y=\"-92.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_layer2_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2078,-30C2078,-30 2664,-30 2664,-30 2670,-30 2676,-36 2676,-42 2676,-42 2676,-142 2676,-142 2676,-148 2670,-154 2664,-154 2664,-154 2078,-154 2078,-154 2072,-154 2066,-148 2066,-142 2066,-142 2066,-42 2066,-42 2066,-36 2072,-30 2078,-30\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-138.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust11\" class=\"cluster\"><title>cluster_layer2_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2258,-38C2258,-38 2574,-38 2574,-38 2580,-38 2586,-44 2586,-50 2586,-50 2586,-101 2586,-101 2586,-107 2580,-113 2574,-113 2574,-113 2258,-113 2258,-113 2252,-113 2246,-107 2246,-101 2246,-101 2246,-50 2246,-50 2246,-44 2252,-38 2258,-38\"/>\n",
       "<text text-anchor=\"middle\" x=\"2416\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust12\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2704,-12C2704,-12 3936,-12 3936,-12 3942,-12 3948,-18 3948,-24 3948,-24 3948,-197 3948,-197 3948,-203 3942,-209 3936,-209 3936,-209 2704,-209 2704,-209 2698,-209 2692,-203 2692,-197 2692,-197 2692,-24 2692,-24 2692,-18 2698,-12 2704,-12\"/>\n",
       "<text text-anchor=\"middle\" x=\"3320\" y=\"-193.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust13\" class=\"cluster\"><title>cluster_layer3_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2712,-20C2712,-20 3298,-20 3298,-20 3304,-20 3310,-26 3310,-32 3310,-32 3310,-166 3310,-166 3310,-172 3304,-178 3298,-178 3298,-178 2712,-178 2712,-178 2706,-178 2700,-172 2700,-166 2700,-166 2700,-32 2700,-32 2700,-26 2706,-20 2712,-20\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-162.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust14\" class=\"cluster\"><title>cluster_layer3_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2892,-28C2892,-28 3208,-28 3208,-28 3214,-28 3220,-34 3220,-40 3220,-40 3220,-91 3220,-91 3220,-97 3214,-103 3208,-103 3208,-103 2892,-103 2892,-103 2886,-103 2880,-97 2880,-91 2880,-91 2880,-40 2880,-40 2880,-34 2886,-28 2892,-28\"/>\n",
       "<text text-anchor=\"middle\" x=\"3050\" y=\"-87.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust15\" class=\"cluster\"><title>cluster_layer3_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3342,-37C3342,-37 3928,-37 3928,-37 3934,-37 3940,-43 3940,-49 3940,-49 3940,-149 3940,-149 3940,-155 3934,-161 3928,-161 3928,-161 3342,-161 3342,-161 3336,-161 3330,-155 3330,-149 3330,-149 3330,-49 3330,-49 3330,-43 3336,-37 3342,-37\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-145.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust16\" class=\"cluster\"><title>cluster_layer3_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3522,-45C3522,-45 3838,-45 3838,-45 3844,-45 3850,-51 3850,-57 3850,-57 3850,-108 3850,-108 3850,-114 3844,-120 3838,-120 3838,-120 3522,-120 3522,-120 3516,-120 3510,-114 3510,-108 3510,-108 3510,-57 3510,-57 3510,-51 3516,-45 3522,-45\"/>\n",
       "<text text-anchor=\"middle\" x=\"3680\" y=\"-104.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust17\" class=\"cluster\"><title>cluster_layer4</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3968,-8C3968,-8 5200,-8 5200,-8 5206,-8 5212,-14 5212,-20 5212,-20 5212,-193 5212,-193 5212,-199 5206,-205 5200,-205 5200,-205 3968,-205 3968,-205 3962,-205 3956,-199 3956,-193 3956,-193 3956,-20 3956,-20 3956,-14 3962,-8 3968,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"4584\" y=\"-189.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer4</text>\n",
       "</g>\n",
       "<g id=\"clust18\" class=\"cluster\"><title>cluster_layer4_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M3976,-16C3976,-16 4562,-16 4562,-16 4568,-16 4574,-22 4574,-28 4574,-28 4574,-162 4574,-162 4574,-168 4568,-174 4562,-174 4562,-174 3976,-174 3976,-174 3970,-174 3964,-168 3964,-162 3964,-162 3964,-28 3964,-28 3964,-22 3970,-16 3976,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-158.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust19\" class=\"cluster\"><title>cluster_layer4_block0_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4156,-24C4156,-24 4472,-24 4472,-24 4478,-24 4484,-30 4484,-36 4484,-36 4484,-87 4484,-87 4484,-93 4478,-99 4472,-99 4472,-99 4156,-99 4156,-99 4150,-99 4144,-93 4144,-87 4144,-87 4144,-36 4144,-36 4144,-30 4150,-24 4156,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"4314\" y=\"-83.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust20\" class=\"cluster\"><title>cluster_layer4_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4606,-33C4606,-33 5192,-33 5192,-33 5198,-33 5204,-39 5204,-45 5204,-45 5204,-145 5204,-145 5204,-151 5198,-157 5192,-157 5192,-157 4606,-157 4606,-157 4600,-157 4594,-151 4594,-145 4594,-145 4594,-45 4594,-45 4594,-39 4600,-33 4606,-33\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-141.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust21\" class=\"cluster\"><title>cluster_layer4_block1_branch</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M4786,-41C4786,-41 5102,-41 5102,-41 5108,-41 5114,-47 5114,-53 5114,-53 5114,-104 5114,-104 5114,-110 5108,-116 5102,-116 5102,-116 4786,-116 4786,-116 4780,-116 4774,-110 4774,-104 4774,-104 4774,-53 4774,-53 4774,-47 4780,-41 4786,-41\"/>\n",
       "<text text-anchor=\"middle\" x=\"4944\" y=\"-100.8\" font-family=\"Times,serif\" font-size=\"14.00\">branch</text>\n",
       "</g>\n",
       "<g id=\"clust22\" class=\"cluster\"><title>cluster_final</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M5236,-34C5236,-34 5653,-34 5653,-34 5659,-34 5665,-40 5665,-46 5665,-46 5665,-151 5665,-151 5665,-157 5659,-163 5653,-163 5653,-163 5236,-163 5236,-163 5230,-163 5224,-157 5224,-151 5224,-151 5224,-46 5224,-46 5224,-40 5230,-34 5236,-34\"/>\n",
       "<text text-anchor=\"middle\" x=\"5444.5\" y=\"-147.8\" font-family=\"Times,serif\" font-size=\"14.00\">final</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-96C132,-96 102,-96 102,-96 96,-96 90,-90 90,-84 90,-84 90,-72 90,-72 90,-66 96,-60 102,-60 102,-60 132,-60 132,-60 138,-60 144,-66 144,-72 144,-72 144,-84 144,-84 144,-90 138,-96 132,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1 -->\n",
       "<g id=\"node2\" class=\"node\"><title>layer1_block0_bn1</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-96C222,-96 192,-96 192,-96 186,-96 180,-90 180,-84 180,-84 180,-72 180,-72 180,-66 186,-60 192,-60 192,-60 222,-60 222,-60 228,-60 234,-66 234,-72 234,-72 234,-84 234,-84 234,-90 228,-96 222,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;layer1_block0_bn1 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;layer1_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-78C152.393,-78 161.311,-78 169.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-81.5001 179.919,-78 169.919,-74.5001 169.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>layer1_block0_relu1</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-96C312,-96 282,-96 282,-96 276,-96 270,-90 270,-84 270,-84 270,-72 270,-72 270,-66 276,-60 282,-60 282,-60 312,-60 312,-60 318,-60 324,-66 324,-72 324,-72 324,-84 324,-84 324,-90 318,-96 312,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1&#45;&gt;layer1_block0_relu1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>layer1_block0_bn1&#45;&gt;layer1_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-78C242.393,-78 251.311,-78 259.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-81.5001 269.919,-78 259.919,-74.5001 259.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_block0_branch_conv1</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-76C402,-76 372,-76 372,-76 366,-76 360,-70 360,-64 360,-64 360,-52 360,-52 360,-46 366,-40 372,-40 372,-40 402,-40 402,-40 408,-40 414,-46 414,-52 414,-52 414,-64 414,-64 414,-70 408,-76 402,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-71.9993C332.481,-70.1634 341.507,-68.1121 350.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"350.943,-69.5568 359.919,-63.9275 349.392,-62.7308 350.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_add -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer1_block0_add</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M762,-96C762,-96 732,-96 732,-96 726,-96 720,-90 720,-84 720,-84 720,-72 720,-72 720,-66 726,-60 732,-60 732,-60 762,-60 762,-60 768,-60 774,-66 774,-72 774,-72 774,-84 774,-84 774,-90 768,-96 762,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"747\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M321.737,-96.1018C330.735,-101.964 341.381,-107.78 352,-111 404.782,-127.006 420.844,-116 476,-116 476,-116 476,-116 568,-116 623.156,-116 639.218,-127.006 692,-111 699.301,-108.786 706.614,-105.345 713.396,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"715.545,-104.295 722.263,-96.1018 711.902,-98.3176 715.545,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2 -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_block0_branch_bn2</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-76C492,-76 462,-76 462,-76 456,-76 450,-70 450,-64 450,-64 450,-52 450,-52 450,-46 456,-40 462,-40 462,-40 492,-40 492,-40 498,-40 504,-46 504,-52 504,-52 504,-64 504,-64 504,-70 498,-76 492,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_block0_branch_conv1&#45;&gt;layer1_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-58C422.393,-58 431.311,-58 439.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-61.5001 449.919,-58 439.919,-54.5001 439.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2 -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer1_block0_branch_relu2</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-76C582,-76 552,-76 552,-76 546,-76 540,-70 540,-64 540,-64 540,-52 540,-52 540,-46 546,-40 552,-40 552,-40 582,-40 582,-40 588,-40 594,-46 594,-52 594,-52 594,-64 594,-64 594,-70 588,-76 582,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_block0_branch_bn2&#45;&gt;layer1_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-58C512.393,-58 521.311,-58 529.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-61.5001 539.919,-58 529.919,-54.5001 529.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2 -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer1_block0_branch_conv2</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M672,-76C672,-76 642,-76 642,-76 636,-76 630,-70 630,-64 630,-64 630,-52 630,-52 630,-46 636,-40 642,-40 642,-40 672,-40 672,-40 678,-40 684,-46 684,-52 684,-52 684,-64 684,-64 684,-70 678,-76 672,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer1_block0_branch_relu2&#45;&gt;layer1_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-58C602.393,-58 611.311,-58 619.824,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-61.5001 629.919,-58 619.919,-54.5001 619.919,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_branch_conv2&#45;&gt;layer1_block0_add -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer1_block0_branch_conv2&#45;&gt;layer1_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.403,-64.0007C692.481,-65.8366 701.507,-67.8879 710.105,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"709.392,-73.2692 719.919,-72.0725 710.943,-66.4432 709.392,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1 -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer1_block1_bn1</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M852,-96C852,-96 822,-96 822,-96 816,-96 810,-90 810,-84 810,-84 810,-72 810,-72 810,-66 816,-60 822,-60 822,-60 852,-60 852,-60 858,-60 864,-66 864,-72 864,-72 864,-84 864,-84 864,-90 858,-96 852,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_add&#45;&gt;layer1_block1_bn1 -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer1_block0_add&#45;&gt;layer1_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.403,-78C782.393,-78 791.311,-78 799.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"799.919,-81.5001 809.919,-78 799.919,-74.5001 799.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1 -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer1_block1_relu1</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M942,-96C942,-96 912,-96 912,-96 906,-96 900,-90 900,-84 900,-84 900,-72 900,-72 900,-66 906,-60 912,-60 912,-60 942,-60 942,-60 948,-60 954,-66 954,-72 954,-72 954,-84 954,-84 954,-90 948,-96 942,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1&#45;&gt;layer1_block1_relu1 -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer1_block1_bn1&#45;&gt;layer1_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.403,-78C872.393,-78 881.311,-78 889.824,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"889.919,-81.5001 899.919,-78 889.919,-74.5001 889.919,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1 -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer1_block1_branch_conv1</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1032,-76C1032,-76 1002,-76 1002,-76 996,-76 990,-70 990,-64 990,-64 990,-52 990,-52 990,-46 996,-40 1002,-40 1002,-40 1032,-40 1032,-40 1038,-40 1044,-46 1044,-52 1044,-52 1044,-64 1044,-64 1044,-70 1038,-76 1032,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1 -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M954.403,-71.9993C962.481,-70.1634 971.507,-68.1121 980.105,-66.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"980.943,-69.5568 989.919,-63.9275 979.392,-62.7308 980.943,-69.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_add -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer1_block1_add</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1392,-96C1392,-96 1362,-96 1362,-96 1356,-96 1350,-90 1350,-84 1350,-84 1350,-72 1350,-72 1350,-66 1356,-60 1362,-60 1362,-60 1392,-60 1392,-60 1398,-60 1404,-66 1404,-72 1404,-72 1404,-84 1404,-84 1404,-90 1398,-96 1392,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1377\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M951.737,-96.1018C960.735,-101.964 971.381,-107.78 982,-111 1034.78,-127.006 1050.84,-116 1106,-116 1106,-116 1106,-116 1198,-116 1253.16,-116 1269.22,-127.006 1322,-111 1329.3,-108.786 1336.61,-105.345 1343.4,-101.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1345.55,-104.295 1352.26,-96.1018 1341.9,-98.3176 1345.55,-104.295\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2 -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer1_block1_branch_bn2</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1122,-76C1122,-76 1092,-76 1092,-76 1086,-76 1080,-70 1080,-64 1080,-64 1080,-52 1080,-52 1080,-46 1086,-40 1092,-40 1092,-40 1122,-40 1122,-40 1128,-40 1134,-46 1134,-52 1134,-52 1134,-64 1134,-64 1134,-70 1128,-76 1122,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2 -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer1_block1_branch_conv1&#45;&gt;layer1_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.4,-58C1052.39,-58 1061.31,-58 1069.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1069.92,-61.5001 1079.92,-58 1069.92,-54.5001 1069.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2 -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer1_block1_branch_relu2</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1212,-76C1212,-76 1182,-76 1182,-76 1176,-76 1170,-70 1170,-64 1170,-64 1170,-52 1170,-52 1170,-46 1176,-40 1182,-40 1182,-40 1212,-40 1212,-40 1218,-40 1224,-46 1224,-52 1224,-52 1224,-64 1224,-64 1224,-70 1218,-76 1212,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2 -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer1_block1_branch_bn2&#45;&gt;layer1_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1134.4,-58C1142.39,-58 1151.31,-58 1159.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1159.92,-61.5001 1169.92,-58 1159.92,-54.5001 1159.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2 -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer1_block1_branch_conv2</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1302,-76C1302,-76 1272,-76 1272,-76 1266,-76 1260,-70 1260,-64 1260,-64 1260,-52 1260,-52 1260,-46 1266,-40 1272,-40 1272,-40 1302,-40 1302,-40 1308,-40 1314,-46 1314,-52 1314,-52 1314,-64 1314,-64 1314,-70 1308,-76 1302,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"1287\" y=\"-54.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2 -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer1_block1_branch_relu2&#45;&gt;layer1_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1224.4,-58C1232.39,-58 1241.31,-58 1249.82,-58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1249.92,-61.5001 1259.92,-58 1249.92,-54.5001 1249.92,-61.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_branch_conv2&#45;&gt;layer1_block1_add -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>layer1_block1_branch_conv2&#45;&gt;layer1_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1314.4,-64.0007C1322.48,-65.8366 1331.51,-67.8879 1340.1,-69.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1339.39,-73.2692 1349.92,-72.0725 1340.94,-66.4432 1339.39,-73.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1 -->\n",
       "<g id=\"node16\" class=\"node\"><title>layer2_block0_bn1</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1486,-96C1486,-96 1456,-96 1456,-96 1450,-96 1444,-90 1444,-84 1444,-84 1444,-72 1444,-72 1444,-66 1450,-60 1456,-60 1456,-60 1486,-60 1486,-60 1492,-60 1498,-66 1498,-72 1498,-72 1498,-84 1498,-84 1498,-90 1492,-96 1486,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1471\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_add&#45;&gt;layer2_block0_bn1 -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>layer1_block1_add&#45;&gt;layer2_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1404.12,-78C1413.34,-78 1423.89,-78 1433.8,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1433.82,-81.5001 1443.82,-78 1433.82,-74.5001 1433.82,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1 -->\n",
       "<g id=\"node17\" class=\"node\"><title>layer2_block0_relu1</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1576,-96C1576,-96 1546,-96 1546,-96 1540,-96 1534,-90 1534,-84 1534,-84 1534,-72 1534,-72 1534,-66 1540,-60 1546,-60 1546,-60 1576,-60 1576,-60 1582,-60 1588,-66 1588,-72 1588,-72 1588,-84 1588,-84 1588,-90 1582,-96 1576,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"1561\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1&#45;&gt;layer2_block0_relu1 -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>layer2_block0_bn1&#45;&gt;layer2_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1498.4,-78C1506.39,-78 1515.31,-78 1523.82,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1523.92,-81.5001 1533.92,-78 1523.92,-74.5001 1523.92,-81.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1 -->\n",
       "<g id=\"node18\" class=\"node\"><title>layer2_block0_branch_conv1</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1666,-77C1666,-77 1636,-77 1636,-77 1630,-77 1624,-71 1624,-65 1624,-65 1624,-53 1624,-53 1624,-47 1630,-41 1636,-41 1636,-41 1666,-41 1666,-41 1672,-41 1678,-47 1678,-53 1678,-53 1678,-65 1678,-65 1678,-71 1672,-77 1666,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1651\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1 -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1588.4,-72.2994C1596.48,-70.5552 1605.51,-68.6065 1614.1,-66.7501\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1614.88,-70.1628 1623.92,-64.6311 1613.41,-63.3205 1614.88,-70.1628\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3 -->\n",
       "<g id=\"node22\" class=\"node\"><title>layer2_block0_conv3</title>\n",
       "<g id=\"a_node22\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-152C1936,-152 1906,-152 1906,-152 1900,-152 1894,-146 1894,-140 1894,-140 1894,-128 1894,-128 1894,-122 1900,-116 1906,-116 1906,-116 1936,-116 1936,-116 1942,-116 1948,-122 1948,-128 1948,-128 1948,-140 1948,-140 1948,-146 1942,-152 1936,-152\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-130.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_conv3 -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1585.19,-96.1046C1594.3,-102.243 1605.16,-108.442 1616,-112 1708.77,-142.443 1825.74,-140.431 1883.83,-136.933\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1884.22,-140.415 1893.97,-136.27 1883.76,-133.43 1884.22,-140.415\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2 -->\n",
       "<g id=\"node19\" class=\"node\"><title>layer2_block0_branch_bn2</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1756,-77C1756,-77 1726,-77 1726,-77 1720,-77 1714,-71 1714,-65 1714,-65 1714,-53 1714,-53 1714,-47 1720,-41 1726,-41 1726,-41 1756,-41 1756,-41 1762,-41 1768,-47 1768,-53 1768,-53 1768,-65 1768,-65 1768,-71 1762,-77 1756,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1741\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2 -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>layer2_block0_branch_conv1&#45;&gt;layer2_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1678.4,-59C1686.39,-59 1695.31,-59 1703.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1703.92,-62.5001 1713.92,-59 1703.92,-55.5001 1703.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2 -->\n",
       "<g id=\"node20\" class=\"node\"><title>layer2_block0_branch_relu2</title>\n",
       "<g id=\"a_node20\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1846,-77C1846,-77 1816,-77 1816,-77 1810,-77 1804,-71 1804,-65 1804,-65 1804,-53 1804,-53 1804,-47 1810,-41 1816,-41 1816,-41 1846,-41 1846,-41 1852,-41 1858,-47 1858,-53 1858,-53 1858,-65 1858,-65 1858,-71 1852,-77 1846,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1831\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2 -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>layer2_block0_branch_bn2&#45;&gt;layer2_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1768.4,-59C1776.39,-59 1785.31,-59 1793.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1793.92,-62.5001 1803.92,-59 1793.92,-55.5001 1793.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2 -->\n",
       "<g id=\"node21\" class=\"node\"><title>layer2_block0_branch_conv2</title>\n",
       "<g id=\"a_node21\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1936,-77C1936,-77 1906,-77 1906,-77 1900,-77 1894,-71 1894,-65 1894,-65 1894,-53 1894,-53 1894,-47 1900,-41 1906,-41 1906,-41 1936,-41 1936,-41 1942,-41 1948,-47 1948,-53 1948,-53 1948,-65 1948,-65 1948,-71 1942,-77 1936,-77\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-55.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2 -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>layer2_block0_branch_relu2&#45;&gt;layer2_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1858.4,-59C1866.39,-59 1875.31,-59 1883.82,-59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1883.92,-62.5001 1893.92,-59 1883.92,-55.5001 1883.92,-62.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_add -->\n",
       "<g id=\"node23\" class=\"node\"><title>layer2_block0_add</title>\n",
       "<g id=\"a_node23\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2026,-97C2026,-97 1996,-97 1996,-97 1990,-97 1984,-91 1984,-85 1984,-85 1984,-73 1984,-73 1984,-67 1990,-61 1996,-61 1996,-61 2026,-61 2026,-61 2032,-61 2038,-67 2038,-73 2038,-73 2038,-85 2038,-85 2038,-91 2032,-97 2026,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2011\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_branch_conv2&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>layer2_block0_branch_conv2&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-65.0007C1956.48,-66.8366 1965.51,-68.8879 1974.1,-70.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1973.39,-74.2692 1983.92,-73.0725 1974.94,-67.4432 1973.39,-74.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3&#45;&gt;layer2_block0_add -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>layer2_block0_conv3&#45;&gt;layer2_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1948.4,-117.498C1956.83,-112.23 1966.29,-106.317 1975.22,-100.735\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1977.29,-103.569 1983.92,-95.3006 1973.58,-97.6327 1977.29,-103.569\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1 -->\n",
       "<g id=\"node24\" class=\"node\"><title>layer2_block1_bn1</title>\n",
       "<g id=\"a_node24\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2116,-97C2116,-97 2086,-97 2086,-97 2080,-97 2074,-91 2074,-85 2074,-85 2074,-73 2074,-73 2074,-67 2080,-61 2086,-61 2086,-61 2116,-61 2116,-61 2122,-61 2128,-67 2128,-73 2128,-73 2128,-85 2128,-85 2128,-91 2122,-97 2116,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2101\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_add&#45;&gt;layer2_block1_bn1 -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>layer2_block0_add&#45;&gt;layer2_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2038.4,-79C2046.39,-79 2055.31,-79 2063.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2063.92,-82.5001 2073.92,-79 2063.92,-75.5001 2063.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1 -->\n",
       "<g id=\"node25\" class=\"node\"><title>layer2_block1_relu1</title>\n",
       "<g id=\"a_node25\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2206,-97C2206,-97 2176,-97 2176,-97 2170,-97 2164,-91 2164,-85 2164,-85 2164,-73 2164,-73 2164,-67 2170,-61 2176,-61 2176,-61 2206,-61 2206,-61 2212,-61 2218,-67 2218,-73 2218,-73 2218,-85 2218,-85 2218,-91 2212,-97 2206,-97\"/>\n",
       "<text text-anchor=\"middle\" x=\"2191\" y=\"-75.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1&#45;&gt;layer2_block1_relu1 -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>layer2_block1_bn1&#45;&gt;layer2_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2128.4,-79C2136.39,-79 2145.31,-79 2153.82,-79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2153.92,-82.5001 2163.92,-79 2153.92,-75.5001 2153.92,-82.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1 -->\n",
       "<g id=\"node26\" class=\"node\"><title>layer2_block1_branch_conv1</title>\n",
       "<g id=\"a_node26\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2296,-82C2296,-82 2266,-82 2266,-82 2260,-82 2254,-76 2254,-70 2254,-70 2254,-58 2254,-58 2254,-52 2260,-46 2266,-46 2266,-46 2296,-46 2296,-46 2302,-46 2308,-52 2308,-58 2308,-58 2308,-70 2308,-70 2308,-76 2302,-82 2296,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2281\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1 -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2218.4,-74.4995C2226.39,-73.1375 2235.31,-71.6175 2243.82,-70.1664\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2244.65,-73.5763 2253.92,-68.4456 2243.47,-66.6758 2244.65,-73.5763\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_add -->\n",
       "<g id=\"node30\" class=\"node\"><title>layer2_block1_add</title>\n",
       "<g id=\"a_node30\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2656,-82C2656,-82 2626,-82 2626,-82 2620,-82 2614,-76 2614,-70 2614,-70 2614,-58 2614,-58 2614,-52 2620,-46 2626,-46 2626,-46 2656,-46 2656,-46 2662,-46 2668,-52 2668,-58 2668,-58 2668,-70 2668,-70 2668,-76 2662,-82 2656,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2641\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge33\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2211.74,-97.0337C2221.35,-104.671 2233.51,-112.769 2246,-117 2298.24,-134.703 2314.84,-122 2370,-122 2370,-122 2370,-122 2462,-122 2517.16,-122 2535.38,-138.893 2586,-117 2599.55,-111.141 2611.6,-100.216 2620.87,-89.7996\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2623.65,-91.9369 2627.39,-82.0259 2618.28,-87.4385 2623.65,-91.9369\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2 -->\n",
       "<g id=\"node27\" class=\"node\"><title>layer2_block1_branch_bn2</title>\n",
       "<g id=\"a_node27\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2386,-82C2386,-82 2356,-82 2356,-82 2350,-82 2344,-76 2344,-70 2344,-70 2344,-58 2344,-58 2344,-52 2350,-46 2356,-46 2356,-46 2386,-46 2386,-46 2392,-46 2398,-52 2398,-58 2398,-58 2398,-70 2398,-70 2398,-76 2392,-82 2386,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2371\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2 -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>layer2_block1_branch_conv1&#45;&gt;layer2_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2308.4,-64C2316.39,-64 2325.31,-64 2333.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2333.92,-67.5001 2343.92,-64 2333.92,-60.5001 2333.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2 -->\n",
       "<g id=\"node28\" class=\"node\"><title>layer2_block1_branch_relu2</title>\n",
       "<g id=\"a_node28\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2476,-82C2476,-82 2446,-82 2446,-82 2440,-82 2434,-76 2434,-70 2434,-70 2434,-58 2434,-58 2434,-52 2440,-46 2446,-46 2446,-46 2476,-46 2476,-46 2482,-46 2488,-52 2488,-58 2488,-58 2488,-70 2488,-70 2488,-76 2482,-82 2476,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2461\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2 -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>layer2_block1_branch_bn2&#45;&gt;layer2_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2398.4,-64C2406.39,-64 2415.31,-64 2423.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2423.92,-67.5001 2433.92,-64 2423.92,-60.5001 2423.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2 -->\n",
       "<g id=\"node29\" class=\"node\"><title>layer2_block1_branch_conv2</title>\n",
       "<g id=\"a_node29\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2566,-82C2566,-82 2536,-82 2536,-82 2530,-82 2524,-76 2524,-70 2524,-70 2524,-58 2524,-58 2524,-52 2530,-46 2536,-46 2536,-46 2566,-46 2566,-46 2572,-46 2578,-52 2578,-58 2578,-58 2578,-70 2578,-70 2578,-76 2572,-82 2566,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2551\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2 -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>layer2_block1_branch_relu2&#45;&gt;layer2_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2488.4,-64C2496.39,-64 2505.31,-64 2513.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2513.92,-67.5001 2523.92,-64 2513.92,-60.5001 2513.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_branch_conv2&#45;&gt;layer2_block1_add -->\n",
       "<g id=\"edge34\" class=\"edge\"><title>layer2_block1_branch_conv2&#45;&gt;layer2_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2578.4,-64C2586.39,-64 2595.31,-64 2603.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2603.92,-67.5001 2613.92,-64 2603.92,-60.5001 2603.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1 -->\n",
       "<g id=\"node31\" class=\"node\"><title>layer3_block0_bn1</title>\n",
       "<g id=\"a_node31\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2750,-82C2750,-82 2720,-82 2720,-82 2714,-82 2708,-76 2708,-70 2708,-70 2708,-58 2708,-58 2708,-52 2714,-46 2720,-46 2720,-46 2750,-46 2750,-46 2756,-46 2762,-52 2762,-58 2762,-58 2762,-70 2762,-70 2762,-76 2756,-82 2750,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2735\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_add&#45;&gt;layer3_block0_bn1 -->\n",
       "<g id=\"edge35\" class=\"edge\"><title>layer2_block1_add&#45;&gt;layer3_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2668.12,-64C2677.34,-64 2687.89,-64 2697.8,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2697.82,-67.5001 2707.82,-64 2697.82,-60.5001 2697.82,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1 -->\n",
       "<g id=\"node32\" class=\"node\"><title>layer3_block0_relu1</title>\n",
       "<g id=\"a_node32\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2840,-82C2840,-82 2810,-82 2810,-82 2804,-82 2798,-76 2798,-70 2798,-70 2798,-58 2798,-58 2798,-52 2804,-46 2810,-46 2810,-46 2840,-46 2840,-46 2846,-46 2852,-52 2852,-58 2852,-58 2852,-70 2852,-70 2852,-76 2846,-82 2840,-82\"/>\n",
       "<text text-anchor=\"middle\" x=\"2825\" y=\"-60.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1&#45;&gt;layer3_block0_relu1 -->\n",
       "<g id=\"edge36\" class=\"edge\"><title>layer3_block0_bn1&#45;&gt;layer3_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2762.4,-64C2770.39,-64 2779.31,-64 2787.82,-64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2787.92,-67.5001 2797.92,-64 2787.92,-60.5001 2787.92,-67.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1 -->\n",
       "<g id=\"node33\" class=\"node\"><title>layer3_block0_branch_conv1</title>\n",
       "<g id=\"a_node33\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2930,-72C2930,-72 2900,-72 2900,-72 2894,-72 2888,-66 2888,-60 2888,-60 2888,-48 2888,-48 2888,-42 2894,-36 2900,-36 2900,-36 2930,-36 2930,-36 2936,-36 2942,-42 2942,-48 2942,-48 2942,-60 2942,-60 2942,-66 2936,-72 2930,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"2915\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1 -->\n",
       "<g id=\"edge37\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2852.4,-60.9997C2860.39,-60.0917 2869.31,-59.0783 2877.82,-58.1109\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2878.38,-61.5706 2887.92,-56.9638 2877.59,-54.6153 2878.38,-61.5706\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3 -->\n",
       "<g id=\"node37\" class=\"node\"><title>layer3_block0_conv3</title>\n",
       "<g id=\"a_node37\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-147C3200,-147 3170,-147 3170,-147 3164,-147 3158,-141 3158,-135 3158,-135 3158,-123 3158,-123 3158,-117 3164,-111 3170,-111 3170,-111 3200,-111 3200,-111 3206,-111 3212,-117 3212,-123 3212,-123 3212,-135 3212,-135 3212,-141 3206,-147 3200,-147\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-125.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_conv3 -->\n",
       "<g id=\"edge41\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2843.22,-82.1953C2853.06,-91.336 2866.16,-101.594 2880,-107 2970.95,-142.522 3088.72,-138.267 3147.4,-133.121\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3148.02,-136.578 3157.65,-132.159 3147.37,-129.609 3148.02,-136.578\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2 -->\n",
       "<g id=\"node34\" class=\"node\"><title>layer3_block0_branch_bn2</title>\n",
       "<g id=\"a_node34\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3020,-72C3020,-72 2990,-72 2990,-72 2984,-72 2978,-66 2978,-60 2978,-60 2978,-48 2978,-48 2978,-42 2984,-36 2990,-36 2990,-36 3020,-36 3020,-36 3026,-36 3032,-42 3032,-48 3032,-48 3032,-60 3032,-60 3032,-66 3026,-72 3020,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3005\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2 -->\n",
       "<g id=\"edge38\" class=\"edge\"><title>layer3_block0_branch_conv1&#45;&gt;layer3_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2942.4,-54C2950.39,-54 2959.31,-54 2967.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2967.92,-57.5001 2977.92,-54 2967.92,-50.5001 2967.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2 -->\n",
       "<g id=\"node35\" class=\"node\"><title>layer3_block0_branch_relu2</title>\n",
       "<g id=\"a_node35\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3110,-72C3110,-72 3080,-72 3080,-72 3074,-72 3068,-66 3068,-60 3068,-60 3068,-48 3068,-48 3068,-42 3074,-36 3080,-36 3080,-36 3110,-36 3110,-36 3116,-36 3122,-42 3122,-48 3122,-48 3122,-60 3122,-60 3122,-66 3116,-72 3110,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3095\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2 -->\n",
       "<g id=\"edge39\" class=\"edge\"><title>layer3_block0_branch_bn2&#45;&gt;layer3_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3032.4,-54C3040.39,-54 3049.31,-54 3057.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3057.92,-57.5001 3067.92,-54 3057.92,-50.5001 3057.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2 -->\n",
       "<g id=\"node36\" class=\"node\"><title>layer3_block0_branch_conv2</title>\n",
       "<g id=\"a_node36\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3200,-72C3200,-72 3170,-72 3170,-72 3164,-72 3158,-66 3158,-60 3158,-60 3158,-48 3158,-48 3158,-42 3164,-36 3170,-36 3170,-36 3200,-36 3200,-36 3206,-36 3212,-42 3212,-48 3212,-48 3212,-60 3212,-60 3212,-66 3206,-72 3200,-72\"/>\n",
       "<text text-anchor=\"middle\" x=\"3185\" y=\"-50.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2 -->\n",
       "<g id=\"edge40\" class=\"edge\"><title>layer3_block0_branch_relu2&#45;&gt;layer3_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3122.4,-54C3130.39,-54 3139.31,-54 3147.82,-54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3147.92,-57.5001 3157.92,-54 3147.92,-50.5001 3147.92,-57.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_add -->\n",
       "<g id=\"node38\" class=\"node\"><title>layer3_block0_add</title>\n",
       "<g id=\"a_node38\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3290,-109C3290,-109 3260,-109 3260,-109 3254,-109 3248,-103 3248,-97 3248,-97 3248,-85 3248,-85 3248,-79 3254,-73 3260,-73 3260,-73 3290,-73 3290,-73 3296,-73 3302,-79 3302,-85 3302,-85 3302,-97 3302,-97 3302,-103 3296,-109 3290,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3275\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_branch_conv2&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge43\" class=\"edge\"><title>layer3_block0_branch_conv2&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-65.1012C3220.66,-68.5716 3229.9,-72.4578 3238.66,-76.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3237.34,-79.3846 3247.92,-80.0341 3240.06,-72.9318 3237.34,-79.3846\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3&#45;&gt;layer3_block0_add -->\n",
       "<g id=\"edge42\" class=\"edge\"><title>layer3_block0_conv3&#45;&gt;layer3_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3212.4,-117.599C3220.66,-114.035 3229.9,-110.043 3238.66,-106.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3240.13,-109.44 3247.92,-102.262 3237.35,-103.013 3240.13,-109.44\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1 -->\n",
       "<g id=\"node39\" class=\"node\"><title>layer3_block1_bn1</title>\n",
       "<g id=\"a_node39\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3380,-109C3380,-109 3350,-109 3350,-109 3344,-109 3338,-103 3338,-97 3338,-97 3338,-85 3338,-85 3338,-79 3344,-73 3350,-73 3350,-73 3380,-73 3380,-73 3386,-73 3392,-79 3392,-85 3392,-85 3392,-97 3392,-97 3392,-103 3386,-109 3380,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3365\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_add&#45;&gt;layer3_block1_bn1 -->\n",
       "<g id=\"edge44\" class=\"edge\"><title>layer3_block0_add&#45;&gt;layer3_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3302.4,-91C3310.39,-91 3319.31,-91 3327.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3327.92,-94.5001 3337.92,-91 3327.92,-87.5001 3327.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1 -->\n",
       "<g id=\"node40\" class=\"node\"><title>layer3_block1_relu1</title>\n",
       "<g id=\"a_node40\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3470,-109C3470,-109 3440,-109 3440,-109 3434,-109 3428,-103 3428,-97 3428,-97 3428,-85 3428,-85 3428,-79 3434,-73 3440,-73 3440,-73 3470,-73 3470,-73 3476,-73 3482,-79 3482,-85 3482,-85 3482,-97 3482,-97 3482,-103 3476,-109 3470,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3455\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1&#45;&gt;layer3_block1_relu1 -->\n",
       "<g id=\"edge45\" class=\"edge\"><title>layer3_block1_bn1&#45;&gt;layer3_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3392.4,-91C3400.39,-91 3409.31,-91 3417.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3417.92,-94.5001 3427.92,-91 3417.92,-87.5001 3417.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1 -->\n",
       "<g id=\"node41\" class=\"node\"><title>layer3_block1_branch_conv1</title>\n",
       "<g id=\"a_node41\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3560,-89C3560,-89 3530,-89 3530,-89 3524,-89 3518,-83 3518,-77 3518,-77 3518,-65 3518,-65 3518,-59 3524,-53 3530,-53 3530,-53 3560,-53 3560,-53 3566,-53 3572,-59 3572,-65 3572,-65 3572,-77 3572,-77 3572,-83 3566,-89 3560,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3545\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1 -->\n",
       "<g id=\"edge46\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3482.4,-84.9993C3490.48,-83.1634 3499.51,-81.1121 3508.1,-79.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3508.94,-82.5568 3517.92,-76.9275 3507.39,-75.7308 3508.94,-82.5568\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_add -->\n",
       "<g id=\"node45\" class=\"node\"><title>layer3_block1_add</title>\n",
       "<g id=\"a_node45\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M3920,-109C3920,-109 3890,-109 3890,-109 3884,-109 3878,-103 3878,-97 3878,-97 3878,-85 3878,-85 3878,-79 3884,-73 3890,-73 3890,-73 3920,-73 3920,-73 3926,-73 3932,-79 3932,-85 3932,-85 3932,-97 3932,-97 3932,-103 3926,-109 3920,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3905\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge50\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3479.74,-109.102C3488.74,-114.964 3499.38,-120.78 3510,-124 3562.78,-140.006 3578.84,-129 3634,-129 3634,-129 3634,-129 3726,-129 3781.16,-129 3797.22,-140.006 3850,-124 3857.3,-121.786 3864.61,-118.345 3871.4,-114.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3873.55,-117.295 3880.26,-109.102 3869.9,-111.318 3873.55,-117.295\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2 -->\n",
       "<g id=\"node42\" class=\"node\"><title>layer3_block1_branch_bn2</title>\n",
       "<g id=\"a_node42\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M3650,-89C3650,-89 3620,-89 3620,-89 3614,-89 3608,-83 3608,-77 3608,-77 3608,-65 3608,-65 3608,-59 3614,-53 3620,-53 3620,-53 3650,-53 3650,-53 3656,-53 3662,-59 3662,-65 3662,-65 3662,-77 3662,-77 3662,-83 3656,-89 3650,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3635\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2 -->\n",
       "<g id=\"edge47\" class=\"edge\"><title>layer3_block1_branch_conv1&#45;&gt;layer3_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3572.4,-71C3580.39,-71 3589.31,-71 3597.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3597.92,-74.5001 3607.92,-71 3597.92,-67.5001 3597.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2 -->\n",
       "<g id=\"node43\" class=\"node\"><title>layer3_block1_branch_relu2</title>\n",
       "<g id=\"a_node43\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M3740,-89C3740,-89 3710,-89 3710,-89 3704,-89 3698,-83 3698,-77 3698,-77 3698,-65 3698,-65 3698,-59 3704,-53 3710,-53 3710,-53 3740,-53 3740,-53 3746,-53 3752,-59 3752,-65 3752,-65 3752,-77 3752,-77 3752,-83 3746,-89 3740,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3725\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2 -->\n",
       "<g id=\"edge48\" class=\"edge\"><title>layer3_block1_branch_bn2&#45;&gt;layer3_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3662.4,-71C3670.39,-71 3679.31,-71 3687.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3687.92,-74.5001 3697.92,-71 3687.92,-67.5001 3687.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2 -->\n",
       "<g id=\"node44\" class=\"node\"><title>layer3_block1_branch_conv2</title>\n",
       "<g id=\"a_node44\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M3830,-89C3830,-89 3800,-89 3800,-89 3794,-89 3788,-83 3788,-77 3788,-77 3788,-65 3788,-65 3788,-59 3794,-53 3800,-53 3800,-53 3830,-53 3830,-53 3836,-53 3842,-59 3842,-65 3842,-65 3842,-77 3842,-77 3842,-83 3836,-89 3830,-89\"/>\n",
       "<text text-anchor=\"middle\" x=\"3815\" y=\"-67.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2 -->\n",
       "<g id=\"edge49\" class=\"edge\"><title>layer3_block1_branch_relu2&#45;&gt;layer3_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3752.4,-71C3760.39,-71 3769.31,-71 3777.82,-71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3777.92,-74.5001 3787.92,-71 3777.92,-67.5001 3777.92,-74.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_branch_conv2&#45;&gt;layer3_block1_add -->\n",
       "<g id=\"edge51\" class=\"edge\"><title>layer3_block1_branch_conv2&#45;&gt;layer3_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3842.4,-77.0007C3850.48,-78.8366 3859.51,-80.8879 3868.1,-82.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3867.39,-86.2692 3877.92,-85.0725 3868.94,-79.4432 3867.39,-86.2692\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1 -->\n",
       "<g id=\"node46\" class=\"node\"><title>layer4_block0_bn1</title>\n",
       "<g id=\"a_node46\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4014,-109C4014,-109 3984,-109 3984,-109 3978,-109 3972,-103 3972,-97 3972,-97 3972,-85 3972,-85 3972,-79 3978,-73 3984,-73 3984,-73 4014,-73 4014,-73 4020,-73 4026,-79 4026,-85 4026,-85 4026,-97 4026,-97 4026,-103 4020,-109 4014,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"3999\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_add&#45;&gt;layer4_block0_bn1 -->\n",
       "<g id=\"edge52\" class=\"edge\"><title>layer3_block1_add&#45;&gt;layer4_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3932.12,-91C3941.34,-91 3951.89,-91 3961.8,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3961.82,-94.5001 3971.82,-91 3961.82,-87.5001 3961.82,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1 -->\n",
       "<g id=\"node47\" class=\"node\"><title>layer4_block0_relu1</title>\n",
       "<g id=\"a_node47\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4104,-109C4104,-109 4074,-109 4074,-109 4068,-109 4062,-103 4062,-97 4062,-97 4062,-85 4062,-85 4062,-79 4068,-73 4074,-73 4074,-73 4104,-73 4104,-73 4110,-73 4116,-79 4116,-85 4116,-85 4116,-97 4116,-97 4116,-103 4110,-109 4104,-109\"/>\n",
       "<text text-anchor=\"middle\" x=\"4089\" y=\"-87.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1&#45;&gt;layer4_block0_relu1 -->\n",
       "<g id=\"edge53\" class=\"edge\"><title>layer4_block0_bn1&#45;&gt;layer4_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4026.4,-91C4034.39,-91 4043.31,-91 4051.82,-91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4051.92,-94.5001 4061.92,-91 4051.92,-87.5001 4051.92,-94.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1 -->\n",
       "<g id=\"node48\" class=\"node\"><title>layer4_block0_branch_conv1</title>\n",
       "<g id=\"a_node48\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4194,-68C4194,-68 4164,-68 4164,-68 4158,-68 4152,-62 4152,-56 4152,-56 4152,-44 4152,-44 4152,-38 4158,-32 4164,-32 4164,-32 4194,-32 4194,-32 4200,-32 4206,-38 4206,-44 4206,-44 4206,-56 4206,-56 4206,-62 4200,-68 4194,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4179\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1 -->\n",
       "<g id=\"edge54\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.4,-78.6986C4124.66,-74.8531 4133.9,-70.5468 4142.66,-66.463\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4144.33,-69.5472 4151.92,-62.1514 4141.38,-63.2021 4144.33,-69.5472\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3 -->\n",
       "<g id=\"node52\" class=\"node\"><title>layer4_block0_conv3</title>\n",
       "<g id=\"a_node52\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-143C4464,-143 4434,-143 4434,-143 4428,-143 4422,-137 4422,-131 4422,-131 4422,-119 4422,-119 4422,-113 4428,-107 4434,-107 4434,-107 4464,-107 4464,-107 4470,-107 4476,-113 4476,-119 4476,-119 4476,-131 4476,-131 4476,-137 4470,-143 4464,-143\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-121.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_conv3 -->\n",
       "<g id=\"edge58\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4116.22,-97.6371C4125,-99.6421 4134.88,-101.664 4144,-103 4239.91,-117.051 4354.23,-122.258 4411.49,-124.083\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.73,-127.592 4421.83,-124.395 4411.94,-120.595 4411.73,-127.592\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2 -->\n",
       "<g id=\"node49\" class=\"node\"><title>layer4_block0_branch_bn2</title>\n",
       "<g id=\"a_node49\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4284,-68C4284,-68 4254,-68 4254,-68 4248,-68 4242,-62 4242,-56 4242,-56 4242,-44 4242,-44 4242,-38 4248,-32 4254,-32 4254,-32 4284,-32 4284,-32 4290,-32 4296,-38 4296,-44 4296,-44 4296,-56 4296,-56 4296,-62 4290,-68 4284,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4269\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2 -->\n",
       "<g id=\"edge55\" class=\"edge\"><title>layer4_block0_branch_conv1&#45;&gt;layer4_block0_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4206.4,-50C4214.39,-50 4223.31,-50 4231.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4231.92,-53.5001 4241.92,-50 4231.92,-46.5001 4231.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2 -->\n",
       "<g id=\"node50\" class=\"node\"><title>layer4_block0_branch_relu2</title>\n",
       "<g id=\"a_node50\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4374,-68C4374,-68 4344,-68 4344,-68 4338,-68 4332,-62 4332,-56 4332,-56 4332,-44 4332,-44 4332,-38 4338,-32 4344,-32 4344,-32 4374,-32 4374,-32 4380,-32 4386,-38 4386,-44 4386,-44 4386,-56 4386,-56 4386,-62 4380,-68 4374,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4359\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2 -->\n",
       "<g id=\"edge56\" class=\"edge\"><title>layer4_block0_branch_bn2&#45;&gt;layer4_block0_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4296.4,-50C4304.39,-50 4313.31,-50 4321.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4321.92,-53.5001 4331.92,-50 4321.92,-46.5001 4321.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2 -->\n",
       "<g id=\"node51\" class=\"node\"><title>layer4_block0_branch_conv2</title>\n",
       "<g id=\"a_node51\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4464,-68C4464,-68 4434,-68 4434,-68 4428,-68 4422,-62 4422,-56 4422,-56 4422,-44 4422,-44 4422,-38 4428,-32 4434,-32 4434,-32 4464,-32 4464,-32 4470,-32 4476,-38 4476,-44 4476,-44 4476,-56 4476,-56 4476,-62 4470,-68 4464,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"4449\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2 -->\n",
       "<g id=\"edge57\" class=\"edge\"><title>layer4_block0_branch_relu2&#45;&gt;layer4_block0_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4386.4,-50C4394.39,-50 4403.31,-50 4411.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4411.92,-53.5001 4421.92,-50 4411.92,-46.5001 4411.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_add -->\n",
       "<g id=\"node53\" class=\"node\"><title>layer4_block0_add</title>\n",
       "<g id=\"a_node53\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M4554,-105C4554,-105 4524,-105 4524,-105 4518,-105 4512,-99 4512,-93 4512,-93 4512,-81 4512,-81 4512,-75 4518,-69 4524,-69 4524,-69 4554,-69 4554,-69 4560,-69 4566,-75 4566,-81 4566,-81 4566,-93 4566,-93 4566,-99 4560,-105 4554,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4539\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_branch_conv2&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge60\" class=\"edge\"><title>layer4_block0_branch_conv2&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-61.1012C4484.66,-64.5716 4493.9,-68.4578 4502.66,-72.1431\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4501.34,-75.3846 4511.92,-76.0341 4504.06,-68.9318 4501.34,-75.3846\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3&#45;&gt;layer4_block0_add -->\n",
       "<g id=\"edge59\" class=\"edge\"><title>layer4_block0_conv3&#45;&gt;layer4_block0_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4476.4,-113.599C4484.66,-110.035 4493.9,-106.043 4502.66,-102.258\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4504.13,-105.44 4511.92,-98.2623 4501.35,-99.0135 4504.13,-105.44\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1 -->\n",
       "<g id=\"node54\" class=\"node\"><title>layer4_block1_bn1</title>\n",
       "<g id=\"a_node54\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4644,-105C4644,-105 4614,-105 4614,-105 4608,-105 4602,-99 4602,-93 4602,-93 4602,-81 4602,-81 4602,-75 4608,-69 4614,-69 4614,-69 4644,-69 4644,-69 4650,-69 4656,-75 4656,-81 4656,-81 4656,-93 4656,-93 4656,-99 4650,-105 4644,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4629\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_add&#45;&gt;layer4_block1_bn1 -->\n",
       "<g id=\"edge61\" class=\"edge\"><title>layer4_block0_add&#45;&gt;layer4_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4566.4,-87C4574.39,-87 4583.31,-87 4591.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4591.92,-90.5001 4601.92,-87 4591.92,-83.5001 4591.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1 -->\n",
       "<g id=\"node55\" class=\"node\"><title>layer4_block1_relu1</title>\n",
       "<g id=\"a_node55\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M4734,-105C4734,-105 4704,-105 4704,-105 4698,-105 4692,-99 4692,-93 4692,-93 4692,-81 4692,-81 4692,-75 4698,-69 4704,-69 4704,-69 4734,-69 4734,-69 4740,-69 4746,-75 4746,-81 4746,-81 4746,-93 4746,-93 4746,-99 4740,-105 4734,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"4719\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1&#45;&gt;layer4_block1_relu1 -->\n",
       "<g id=\"edge62\" class=\"edge\"><title>layer4_block1_bn1&#45;&gt;layer4_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4656.4,-87C4664.39,-87 4673.31,-87 4681.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4681.92,-90.5001 4691.92,-87 4681.92,-83.5001 4681.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1 -->\n",
       "<g id=\"node56\" class=\"node\"><title>layer4_block1_branch_conv1</title>\n",
       "<g id=\"a_node56\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M4824,-85C4824,-85 4794,-85 4794,-85 4788,-85 4782,-79 4782,-73 4782,-73 4782,-61 4782,-61 4782,-55 4788,-49 4794,-49 4794,-49 4824,-49 4824,-49 4830,-49 4836,-55 4836,-61 4836,-61 4836,-73 4836,-73 4836,-79 4830,-85 4824,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4809\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1 -->\n",
       "<g id=\"edge63\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_branch_conv1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4746.4,-80.9993C4754.48,-79.1634 4763.51,-77.1121 4772.1,-75.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4772.94,-78.5568 4781.92,-72.9275 4771.39,-71.7308 4772.94,-78.5568\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_add -->\n",
       "<g id=\"node60\" class=\"node\"><title>layer4_block1_add</title>\n",
       "<g id=\"a_node60\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M5184,-105C5184,-105 5154,-105 5154,-105 5148,-105 5142,-99 5142,-93 5142,-93 5142,-81 5142,-81 5142,-75 5148,-69 5154,-69 5154,-69 5184,-69 5184,-69 5190,-69 5196,-75 5196,-81 5196,-81 5196,-93 5196,-93 5196,-99 5190,-105 5184,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5169\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge67\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4743.74,-105.102C4752.74,-110.964 4763.38,-116.78 4774,-120 4826.78,-136.006 4842.84,-125 4898,-125 4898,-125 4898,-125 4990,-125 5045.16,-125 5061.22,-136.006 5114,-120 5121.3,-117.786 5128.61,-114.345 5135.4,-110.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5137.55,-113.295 5144.26,-105.102 5133.9,-107.318 5137.55,-113.295\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2 -->\n",
       "<g id=\"node57\" class=\"node\"><title>layer4_block1_branch_bn2</title>\n",
       "<g id=\"a_node57\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M4914,-85C4914,-85 4884,-85 4884,-85 4878,-85 4872,-79 4872,-73 4872,-73 4872,-61 4872,-61 4872,-55 4878,-49 4884,-49 4884,-49 4914,-49 4914,-49 4920,-49 4926,-55 4926,-61 4926,-61 4926,-73 4926,-73 4926,-79 4920,-85 4914,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4899\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2 -->\n",
       "<g id=\"edge64\" class=\"edge\"><title>layer4_block1_branch_conv1&#45;&gt;layer4_block1_branch_bn2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4836.4,-67C4844.39,-67 4853.31,-67 4861.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4861.92,-70.5001 4871.92,-67 4861.92,-63.5001 4861.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2 -->\n",
       "<g id=\"node58\" class=\"node\"><title>layer4_block1_branch_relu2</title>\n",
       "<g id=\"a_node58\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M5004,-85C5004,-85 4974,-85 4974,-85 4968,-85 4962,-79 4962,-73 4962,-73 4962,-61 4962,-61 4962,-55 4968,-49 4974,-49 4974,-49 5004,-49 5004,-49 5010,-49 5016,-55 5016,-61 5016,-61 5016,-73 5016,-73 5016,-79 5010,-85 5004,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"4989\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2 -->\n",
       "<g id=\"edge65\" class=\"edge\"><title>layer4_block1_branch_bn2&#45;&gt;layer4_block1_branch_relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M4926.4,-67C4934.39,-67 4943.31,-67 4951.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"4951.92,-70.5001 4961.92,-67 4951.92,-63.5001 4951.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2 -->\n",
       "<g id=\"node59\" class=\"node\"><title>layer4_block1_branch_conv2</title>\n",
       "<g id=\"a_node59\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M5094,-85C5094,-85 5064,-85 5064,-85 5058,-85 5052,-79 5052,-73 5052,-73 5052,-61 5052,-61 5052,-55 5058,-49 5064,-49 5064,-49 5094,-49 5094,-49 5100,-49 5106,-55 5106,-61 5106,-61 5106,-73 5106,-73 5106,-79 5100,-85 5094,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"5079\" y=\"-63.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2 -->\n",
       "<g id=\"edge66\" class=\"edge\"><title>layer4_block1_branch_relu2&#45;&gt;layer4_block1_branch_conv2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5016.4,-67C5024.39,-67 5033.31,-67 5041.82,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5041.92,-70.5001 5051.92,-67 5041.92,-63.5001 5041.92,-70.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_branch_conv2&#45;&gt;layer4_block1_add -->\n",
       "<g id=\"edge68\" class=\"edge\"><title>layer4_block1_branch_conv2&#45;&gt;layer4_block1_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5106.4,-73.0007C5114.48,-74.8366 5123.51,-76.8879 5132.1,-78.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5131.39,-82.2692 5141.92,-81.0725 5132.94,-75.4432 5131.39,-82.2692\"/>\n",
       "</g>\n",
       "<!-- final_in -->\n",
       "<g id=\"node61\" class=\"node\"><title>final_in</title>\n",
       "<g id=\"a_node61\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5274,-105C5274,-105 5244,-105 5244,-105 5238,-105 5232,-99 5232,-93 5232,-93 5232,-81 5232,-81 5232,-75 5238,-69 5244,-69 5244,-69 5274,-69 5274,-69 5280,-69 5286,-75 5286,-81 5286,-81 5286,-93 5286,-93 5286,-99 5280,-105 5274,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5259\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">in</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_add&#45;&gt;final_in -->\n",
       "<g id=\"edge69\" class=\"edge\"><title>layer4_block1_add&#45;&gt;final_in</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5196.4,-87C5204.39,-87 5213.31,-87 5221.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5221.92,-90.5001 5231.92,-87 5221.92,-83.5001 5221.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_maxpool -->\n",
       "<g id=\"node62\" class=\"node\"><title>final_maxpool</title>\n",
       "<g id=\"a_node62\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#fdb462\" stroke=\"black\" d=\"M5375,-78C5375,-78 5334,-78 5334,-78 5328,-78 5322,-72 5322,-66 5322,-66 5322,-54 5322,-54 5322,-48 5328,-42 5334,-42 5334,-42 5375,-42 5375,-42 5381,-42 5387,-48 5387,-54 5387,-54 5387,-66 5387,-66 5387,-72 5381,-78 5375,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">maxpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_maxpool -->\n",
       "<g id=\"edge70\" class=\"edge\"><title>final_in&#45;&gt;final_maxpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-79.479C5294.13,-77.1436 5303.26,-74.509 5312.12,-71.9497\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.31,-75.2492 5321.94,-69.1121 5311.37,-68.524 5313.31,-75.2492\"/>\n",
       "</g>\n",
       "<!-- final_avgpool -->\n",
       "<g id=\"node63\" class=\"node\"><title>final_avgpool</title>\n",
       "<g id=\"a_node63\"><a xlink:title=\"AvgPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;ceil_mode&#39;: False, &#39;count_include_pad&#39;: True}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M5372.5,-132C5372.5,-132 5336.5,-132 5336.5,-132 5330.5,-132 5324.5,-126 5324.5,-120 5324.5,-120 5324.5,-108 5324.5,-108 5324.5,-102 5330.5,-96 5336.5,-96 5336.5,-96 5372.5,-96 5372.5,-96 5378.5,-96 5384.5,-102 5384.5,-108 5384.5,-108 5384.5,-120 5384.5,-120 5384.5,-126 5378.5,-132 5372.5,-132\"/>\n",
       "<text text-anchor=\"middle\" x=\"5354.5\" y=\"-110.3\" font-family=\"Times,serif\" font-size=\"14.00\">avgpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_avgpool -->\n",
       "<g id=\"edge71\" class=\"edge\"><title>final_in&#45;&gt;final_avgpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5286.04,-94.521C5294.87,-97.0688 5304.92,-99.9726 5314.53,-102.746\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5313.73,-106.157 5324.3,-105.569 5315.67,-99.4322 5313.73,-106.157\"/>\n",
       "</g>\n",
       "<!-- final_concat -->\n",
       "<g id=\"node64\" class=\"node\"><title>final_concat</title>\n",
       "<g id=\"a_node64\"><a xlink:title=\"Concat {}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M5465,-105C5465,-105 5435,-105 5435,-105 5429,-105 5423,-99 5423,-93 5423,-93 5423,-81 5423,-81 5423,-75 5429,-69 5435,-69 5435,-69 5465,-69 5465,-69 5471,-69 5477,-75 5477,-81 5477,-81 5477,-93 5477,-93 5477,-99 5471,-105 5465,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5450\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">concat</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_maxpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge72\" class=\"edge\"><title>final_maxpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5387.15,-69.141C5395.53,-71.5601 5404.63,-74.1884 5413.21,-76.6659\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5412.39,-80.0717 5422.97,-79.4835 5414.33,-73.3465 5412.39,-80.0717\"/>\n",
       "</g>\n",
       "<!-- final_avgpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge73\" class=\"edge\"><title>final_avgpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5384.57,-105.605C5393.55,-103.013 5403.54,-100.127 5412.93,-97.4155\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5414.17,-100.702 5422.8,-94.5648 5412.22,-93.9766 5414.17,-100.702\"/>\n",
       "</g>\n",
       "<!-- final_flatten -->\n",
       "<g id=\"node65\" class=\"node\"><title>final_flatten</title>\n",
       "<g id=\"a_node65\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M5555,-105C5555,-105 5525,-105 5525,-105 5519,-105 5513,-99 5513,-93 5513,-93 5513,-81 5513,-81 5513,-75 5519,-69 5525,-69 5525,-69 5555,-69 5555,-69 5561,-69 5567,-75 5567,-81 5567,-81 5567,-93 5567,-93 5567,-99 5561,-105 5555,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5540\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_concat&#45;&gt;final_flatten -->\n",
       "<g id=\"edge74\" class=\"edge\"><title>final_concat&#45;&gt;final_flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5477.4,-87C5485.39,-87 5494.31,-87 5502.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5502.92,-90.5001 5512.92,-87 5502.92,-83.5001 5502.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- final_linear -->\n",
       "<g id=\"node66\" class=\"node\"><title>final_linear</title>\n",
       "<g id=\"a_node66\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: Parameter containing:\\ntensor([ 0.0125, &#45;0.0352, &#160;0.0155, &#45;0.0063, &#45;0.0125, &#160;0.0192, &#45;0.0177, &#160;0.0126,\\n &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;0.0337, &#160;0.0152], requires_grad=True)}\">\n",
       "<path fill=\"#ccebc5\" stroke=\"black\" d=\"M5645,-105C5645,-105 5615,-105 5615,-105 5609,-105 5603,-99 5603,-93 5603,-93 5603,-81 5603,-81 5603,-75 5609,-69 5615,-69 5615,-69 5645,-69 5645,-69 5651,-69 5657,-75 5657,-81 5657,-81 5657,-93 5657,-93 5657,-99 5651,-105 5645,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5630\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_flatten&#45;&gt;final_linear -->\n",
       "<g id=\"edge75\" class=\"edge\"><title>final_flatten&#45;&gt;final_linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5567.4,-87C5575.39,-87 5584.31,-87 5592.82,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5592.92,-90.5001 5602.92,-87 5592.92,-83.5001 5592.92,-90.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node67\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node67\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M5746,-105C5746,-105 5705,-105 5705,-105 5699,-105 5693,-99 5693,-93 5693,-93 5693,-81 5693,-81 5693,-75 5699,-69 5705,-69 5705,-69 5746,-69 5746,-69 5752,-69 5758,-75 5758,-81 5758,-81 5758,-93 5758,-93 5758,-99 5752,-105 5746,-105\"/>\n",
       "<text text-anchor=\"middle\" x=\"5725.5\" y=\"-83.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_linear&#45;&gt;classifier -->\n",
       "<g id=\"edge76\" class=\"edge\"><title>final_linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M5657.04,-87C5665.05,-87 5674.06,-87 5682.83,-87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"5682.95,-90.5001 5692.94,-87 5682.94,-83.5001 5682.95,-90.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node68\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-96C42,-96 12,-96 12,-96 6,-96 0,-90 0,-84 0,-84 0,-72 0,-72 0,-66 6,-60 12,-60 12,-60 42,-60 42,-60 48,-60 54,-66 54,-72 54,-72 54,-84 54,-84 54,-90 48,-96 42,-96\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-74.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-78C62.3932,-78 71.3106,-78 79.8241,-78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-81.5001 89.919,-78 79.919,-74.5001 79.919,-81.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7f0101a8e550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.0750       5.8285       1.7852       0.3372       0.2924       2.1064       0.3740       6.1209\n",
      "           2       0.1500       4.3309       1.2310       0.5556       0.2903       1.5450       0.5099      10.7422\n",
      "           3       0.2250       4.3070       0.9326       0.6697       0.2909       1.2673       0.5853      15.3401\n",
      "           4       0.3000       4.3083       0.7545       0.7347       0.2922       0.7737       0.7294      19.9407\n",
      "           5       0.3750       4.3424       0.6411       0.7763       0.2922       0.8359       0.7106      24.5753\n",
      "           6       0.4500       4.3339       0.5808       0.7990       0.2903       0.9704       0.6815      29.1995\n",
      "           7       0.5250       4.2997       0.5486       0.8101       0.2891       0.9453       0.7033      33.7883\n",
      "           8       0.6000       4.2997       0.5295       0.8182       0.2925       1.0664       0.6676      38.3806\n",
      "           9       0.5727       4.3297       0.5156       0.8208       0.2895       1.3525       0.6111      42.9997\n",
      "          10       0.5455       4.2983       0.4817       0.8319       0.2909       0.6264       0.7875      47.5889\n",
      "          11       0.5182       4.3097       0.4438       0.8465       0.2915       0.8130       0.7490      52.1902\n",
      "          12       0.4909       4.2983       0.4265       0.8519       0.2897       0.7990       0.7417      56.7783\n",
      "          13       0.4636       4.2944       0.3998       0.8610       0.2893       0.6193       0.7974      61.3620\n",
      "          14       0.4364       4.2984       0.3861       0.8669       0.2903       0.7044       0.7745      65.9507\n",
      "          15       0.4091       4.2933       0.3696       0.8708       0.3152       0.5154       0.8249      70.5592\n",
      "          16       0.3818       4.2911       0.3499       0.8782       0.2915       0.4886       0.8328      75.1418\n",
      "          17       0.3545       4.2939       0.3348       0.8833       0.2911       0.5824       0.8125      79.7268\n",
      "          18       0.3273       4.2945       0.3212       0.8890       0.2907       0.7801       0.7588      84.3120\n",
      "          19       0.3000       4.2973       0.2992       0.8955       0.2920       0.5198       0.8334      88.9012\n",
      "          20       0.2727       4.3196       0.2828       0.9017       0.2914       0.4737       0.8407      93.5122\n",
      "          21       0.2455       4.2935       0.2705       0.9057       0.2891       0.5126       0.8322      98.0949\n",
      "          22       0.2182       4.3143       0.2540       0.9110       0.2922       0.4344       0.8599     102.7014\n",
      "          23       0.1909       4.2949       0.2296       0.9206       0.2891       0.3625       0.8790     107.2854\n",
      "          24       0.1636       4.2896       0.2115       0.9264       0.2901       0.3524       0.8920     111.8651\n",
      "          25       0.1364       4.3189       0.1950       0.9329       0.2911       0.3149       0.8966     116.4751\n",
      "          26       0.1091       4.3194       0.1674       0.9427       0.2906       0.2788       0.9104     121.0852\n",
      "          27       0.0818       4.2957       0.1452       0.9493       0.2908       0.2745       0.9123     125.6717\n",
      "          28       0.0545       4.2961       0.1207       0.9580       0.2914       0.2300       0.9263     130.2592\n",
      "          29       0.0273       4.3185       0.0971       0.9662       0.2913       0.2157       0.9314     134.8690\n",
      "          30       0.0000       4.2925       0.0786       0.9733       0.2899       0.1967       0.9360     139.4514\n"
     ]
    }
   ],
   "source": [
    "lr_schedule = PiecewiseLinear([0, 8, 30], [0, 0.6, 0])\n",
    "batch_size = 768\n",
    "\n",
    "n = DAWN_net()\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(8,8)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 4: Architecture](https://www.myrtle.ai/2018/10/26/how_to_train_your_resnet_4/)  - backbone (36s; test acc 55.9%)\n",
    "\n",
    "It seems reasonable to study how the shortest path through the network trains in isolation and to take steps to improve this before adding back the longer branches. \n",
    "Eliminating the long branches yields the following backbone network in which all convolutions, except for the initial one, have a stride of two.\n",
    "\n",
    "Training the shortest path network for 20 epochs yields an unimpressive test accuracy of 55.9% in 36 seconds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"68pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 67.83\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.44335 0.44335) rotate(0) translate(4 149)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-149 2432,-149 2432,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-21C94,-21 140,-21 140,-21 146,-21 152,-27 152,-33 152,-33 152,-84 152,-84 152,-90 146,-96 140,-96 140,-96 94,-96 94,-96 88,-96 82,-90 82,-84 82,-84 82,-33 82,-33 82,-27 88,-21 94,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M176,-13C176,-13 508,-13 508,-13 514,-13 520,-19 520,-25 520,-25 520,-115 520,-115 520,-121 514,-127 508,-127 508,-127 176,-127 176,-127 170,-127 164,-121 164,-115 164,-115 164,-25 164,-25 164,-19 170,-13 176,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"342\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer1_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M184,-21C184,-21 320,-21 320,-21 326,-21 332,-27 332,-33 332,-33 332,-84 332,-84 332,-90 326,-96 320,-96 320,-96 184,-96 184,-96 178,-96 172,-90 172,-84 172,-84 172,-33 172,-33 172,-27 178,-21 184,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"252\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer1_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-21C364,-21 500,-21 500,-21 506,-21 512,-27 512,-33 512,-33 512,-84 512,-84 512,-90 506,-96 500,-96 500,-96 364,-96 364,-96 358,-96 352,-90 352,-84 352,-84 352,-33 352,-33 352,-27 358,-21 364,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"432\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M540,-13C540,-13 962,-13 962,-13 968,-13 974,-19 974,-25 974,-25 974,-115 974,-115 974,-121 968,-127 962,-127 962,-127 540,-127 540,-127 534,-127 528,-121 528,-115 528,-115 528,-25 528,-25 528,-19 534,-13 540,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"751\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer2_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M548,-21C548,-21 774,-21 774,-21 780,-21 786,-27 786,-33 786,-33 786,-84 786,-84 786,-90 780,-96 774,-96 774,-96 548,-96 548,-96 542,-96 536,-90 536,-84 536,-84 536,-33 536,-33 536,-27 542,-21 548,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"661\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer2_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M818,-21C818,-21 954,-21 954,-21 960,-21 966,-27 966,-33 966,-33 966,-84 966,-84 966,-90 960,-96 954,-96 954,-96 818,-96 818,-96 812,-96 806,-90 806,-84 806,-84 806,-33 806,-33 806,-27 812,-21 818,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"886\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M994,-13C994,-13 1416,-13 1416,-13 1422,-13 1428,-19 1428,-25 1428,-25 1428,-115 1428,-115 1428,-121 1422,-127 1416,-127 1416,-127 994,-127 994,-127 988,-127 982,-121 982,-115 982,-115 982,-25 982,-25 982,-19 988,-13 994,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"1205\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_layer3_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1002,-21C1002,-21 1228,-21 1228,-21 1234,-21 1240,-27 1240,-33 1240,-33 1240,-84 1240,-84 1240,-90 1234,-96 1228,-96 1228,-96 1002,-96 1002,-96 996,-96 990,-90 990,-84 990,-84 990,-33 990,-33 990,-27 996,-21 1002,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"1115\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_layer3_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1272,-21C1272,-21 1408,-21 1408,-21 1414,-21 1420,-27 1420,-33 1420,-33 1420,-84 1420,-84 1420,-90 1414,-96 1408,-96 1408,-96 1272,-96 1272,-96 1266,-96 1260,-90 1260,-84 1260,-84 1260,-33 1260,-33 1260,-27 1266,-21 1272,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"1340\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust11\" class=\"cluster\"><title>cluster_layer4</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1448,-13C1448,-13 1870,-13 1870,-13 1876,-13 1882,-19 1882,-25 1882,-25 1882,-115 1882,-115 1882,-121 1876,-127 1870,-127 1870,-127 1448,-127 1448,-127 1442,-127 1436,-121 1436,-115 1436,-115 1436,-25 1436,-25 1436,-19 1442,-13 1448,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"1659\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer4</text>\n",
       "</g>\n",
       "<g id=\"clust12\" class=\"cluster\"><title>cluster_layer4_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1456,-21C1456,-21 1682,-21 1682,-21 1688,-21 1694,-27 1694,-33 1694,-33 1694,-84 1694,-84 1694,-90 1688,-96 1682,-96 1682,-96 1456,-96 1456,-96 1450,-96 1444,-90 1444,-84 1444,-84 1444,-33 1444,-33 1444,-27 1450,-21 1456,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"1569\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust13\" class=\"cluster\"><title>cluster_layer4_block1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1726,-21C1726,-21 1862,-21 1862,-21 1868,-21 1874,-27 1874,-33 1874,-33 1874,-84 1874,-84 1874,-90 1868,-96 1862,-96 1862,-96 1726,-96 1726,-96 1720,-96 1714,-90 1714,-84 1714,-84 1714,-33 1714,-33 1714,-27 1720,-21 1726,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"1794\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block1</text>\n",
       "</g>\n",
       "<g id=\"clust14\" class=\"cluster\"><title>cluster_final</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1906,-8C1906,-8 2323,-8 2323,-8 2329,-8 2335,-14 2335,-20 2335,-20 2335,-125 2335,-125 2335,-131 2329,-137 2323,-137 2323,-137 1906,-137 1906,-137 1900,-137 1894,-131 1894,-125 1894,-125 1894,-20 1894,-20 1894,-14 1900,-8 1906,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"2114.5\" y=\"-121.8\" font-family=\"Times,serif\" font-size=\"14.00\">final</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-65C132,-65 102,-65 102,-65 96,-65 90,-59 90,-53 90,-53 90,-41 90,-41 90,-35 96,-29 102,-29 102,-29 132,-29 132,-29 138,-29 144,-35 144,-41 144,-41 144,-53 144,-53 144,-59 138,-65 132,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1 -->\n",
       "<g id=\"node2\" class=\"node\"><title>layer1_block0_bn1</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-65C222,-65 192,-65 192,-65 186,-65 180,-59 180,-53 180,-53 180,-41 180,-41 180,-35 186,-29 192,-29 192,-29 222,-29 222,-29 228,-29 234,-35 234,-41 234,-41 234,-53 234,-53 234,-59 228,-65 222,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;layer1_block0_bn1 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;layer1_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-47C152.393,-47 161.311,-47 169.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-50.5001 179.919,-47 169.919,-43.5001 169.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>layer1_block0_relu1</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-65C312,-65 282,-65 282,-65 276,-65 270,-59 270,-53 270,-53 270,-41 270,-41 270,-35 276,-29 282,-29 282,-29 312,-29 312,-29 318,-29 324,-35 324,-41 324,-41 324,-53 324,-53 324,-59 318,-65 312,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_bn1&#45;&gt;layer1_block0_relu1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>layer1_block0_bn1&#45;&gt;layer1_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-47C242.393,-47 251.311,-47 259.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-50.5001 269.919,-47 259.919,-43.5001 259.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_block1_bn1</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M402,-65C402,-65 372,-65 372,-65 366,-65 360,-59 360,-53 360,-53 360,-41 360,-41 360,-35 366,-29 372,-29 372,-29 402,-29 402,-29 408,-29 414,-35 414,-41 414,-41 414,-53 414,-53 414,-59 408,-65 402,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block0_relu1&#45;&gt;layer1_block1_bn1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>layer1_block0_relu1&#45;&gt;layer1_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-47C332.393,-47 341.311,-47 349.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.919,-50.5001 359.919,-47 349.919,-43.5001 349.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1 -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_block1_relu1</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M492,-65C492,-65 462,-65 462,-65 456,-65 450,-59 450,-53 450,-53 450,-41 450,-41 450,-35 456,-29 462,-29 462,-29 492,-29 492,-29 498,-29 504,-35 504,-41 504,-41 504,-53 504,-53 504,-59 498,-65 492,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_bn1&#45;&gt;layer1_block1_relu1 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_block1_bn1&#45;&gt;layer1_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-47C422.393,-47 431.311,-47 439.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-50.5001 449.919,-47 439.919,-43.5001 439.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1 -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer2_block0_bn1</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M586,-65C586,-65 556,-65 556,-65 550,-65 544,-59 544,-53 544,-53 544,-41 544,-41 544,-35 550,-29 556,-29 556,-29 586,-29 586,-29 592,-29 598,-35 598,-41 598,-41 598,-53 598,-53 598,-59 592,-65 586,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"571\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_block1_relu1&#45;&gt;layer2_block0_bn1 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_block1_relu1&#45;&gt;layer2_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.117,-47C513.343,-47 523.891,-47 533.802,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"533.817,-50.5001 543.817,-47 533.817,-43.5001 533.817,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1 -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer2_block0_relu1</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M676,-65C676,-65 646,-65 646,-65 640,-65 634,-59 634,-53 634,-53 634,-41 634,-41 634,-35 640,-29 646,-29 646,-29 676,-29 676,-29 682,-29 688,-35 688,-41 688,-41 688,-53 688,-53 688,-59 682,-65 676,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"661\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_bn1&#45;&gt;layer2_block0_relu1 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer2_block0_bn1&#45;&gt;layer2_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M598.403,-47C606.393,-47 615.311,-47 623.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"623.919,-50.5001 633.919,-47 623.919,-43.5001 623.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3 -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer2_block0_conv3</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M766,-65C766,-65 736,-65 736,-65 730,-65 724,-59 724,-53 724,-53 724,-41 724,-41 724,-35 730,-29 736,-29 736,-29 766,-29 766,-29 772,-29 778,-35 778,-41 778,-41 778,-53 778,-53 778,-59 772,-65 766,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"751\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu1&#45;&gt;layer2_block0_conv3 -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer2_block0_relu1&#45;&gt;layer2_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M688.403,-47C696.393,-47 705.311,-47 713.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"713.919,-50.5001 723.919,-47 713.919,-43.5001 713.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1 -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer2_block1_bn1</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M856,-65C856,-65 826,-65 826,-65 820,-65 814,-59 814,-53 814,-53 814,-41 814,-41 814,-35 820,-29 826,-29 826,-29 856,-29 856,-29 862,-29 868,-35 868,-41 868,-41 868,-53 868,-53 868,-59 862,-65 856,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"841\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_conv3&#45;&gt;layer2_block1_bn1 -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer2_block0_conv3&#45;&gt;layer2_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M778.403,-47C786.393,-47 795.311,-47 803.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"803.919,-50.5001 813.919,-47 803.919,-43.5001 803.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1 -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer2_block1_relu1</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M946,-65C946,-65 916,-65 916,-65 910,-65 904,-59 904,-53 904,-53 904,-41 904,-41 904,-35 910,-29 916,-29 916,-29 946,-29 946,-29 952,-29 958,-35 958,-41 958,-41 958,-53 958,-53 958,-59 952,-65 946,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"931\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_bn1&#45;&gt;layer2_block1_relu1 -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer2_block1_bn1&#45;&gt;layer2_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M868.403,-47C876.393,-47 885.311,-47 893.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"893.919,-50.5001 903.919,-47 893.919,-43.5001 893.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1 -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer3_block0_bn1</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1040,-65C1040,-65 1010,-65 1010,-65 1004,-65 998,-59 998,-53 998,-53 998,-41 998,-41 998,-35 1004,-29 1010,-29 1010,-29 1040,-29 1040,-29 1046,-29 1052,-35 1052,-41 1052,-41 1052,-53 1052,-53 1052,-59 1046,-65 1040,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1025\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block1_relu1&#45;&gt;layer3_block0_bn1 -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer2_block1_relu1&#45;&gt;layer3_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M958.117,-47C967.343,-47 977.891,-47 987.802,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"987.817,-50.5001 997.817,-47 987.817,-43.5001 987.817,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1 -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer3_block0_relu1</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1130,-65C1130,-65 1100,-65 1100,-65 1094,-65 1088,-59 1088,-53 1088,-53 1088,-41 1088,-41 1088,-35 1094,-29 1100,-29 1100,-29 1130,-29 1130,-29 1136,-29 1142,-35 1142,-41 1142,-41 1142,-53 1142,-53 1142,-59 1136,-65 1130,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1115\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_bn1&#45;&gt;layer3_block0_relu1 -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer3_block0_bn1&#45;&gt;layer3_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1052.4,-47C1060.39,-47 1069.31,-47 1077.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1077.92,-50.5001 1087.92,-47 1077.92,-43.5001 1077.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3 -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer3_block0_conv3</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1220,-65C1220,-65 1190,-65 1190,-65 1184,-65 1178,-59 1178,-53 1178,-53 1178,-41 1178,-41 1178,-35 1184,-29 1190,-29 1190,-29 1220,-29 1220,-29 1226,-29 1232,-35 1232,-41 1232,-41 1232,-53 1232,-53 1232,-59 1226,-65 1220,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1205\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu1&#45;&gt;layer3_block0_conv3 -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer3_block0_relu1&#45;&gt;layer3_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1142.4,-47C1150.39,-47 1159.31,-47 1167.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1167.92,-50.5001 1177.92,-47 1167.92,-43.5001 1167.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1 -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer3_block1_bn1</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1310,-65C1310,-65 1280,-65 1280,-65 1274,-65 1268,-59 1268,-53 1268,-53 1268,-41 1268,-41 1268,-35 1274,-29 1280,-29 1280,-29 1310,-29 1310,-29 1316,-29 1322,-35 1322,-41 1322,-41 1322,-53 1322,-53 1322,-59 1316,-65 1310,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1295\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_conv3&#45;&gt;layer3_block1_bn1 -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer3_block0_conv3&#45;&gt;layer3_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1232.4,-47C1240.39,-47 1249.31,-47 1257.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1257.92,-50.5001 1267.92,-47 1257.92,-43.5001 1257.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1 -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer3_block1_relu1</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1400,-65C1400,-65 1370,-65 1370,-65 1364,-65 1358,-59 1358,-53 1358,-53 1358,-41 1358,-41 1358,-35 1364,-29 1370,-29 1370,-29 1400,-29 1400,-29 1406,-29 1412,-35 1412,-41 1412,-41 1412,-53 1412,-53 1412,-59 1406,-65 1400,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1385\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_bn1&#45;&gt;layer3_block1_relu1 -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer3_block1_bn1&#45;&gt;layer3_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1322.4,-47C1330.39,-47 1339.31,-47 1347.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1347.92,-50.5001 1357.92,-47 1347.92,-43.5001 1347.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1 -->\n",
       "<g id=\"node16\" class=\"node\"><title>layer4_block0_bn1</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1494,-65C1494,-65 1464,-65 1464,-65 1458,-65 1452,-59 1452,-53 1452,-53 1452,-41 1452,-41 1452,-35 1458,-29 1464,-29 1464,-29 1494,-29 1494,-29 1500,-29 1506,-35 1506,-41 1506,-41 1506,-53 1506,-53 1506,-59 1500,-65 1494,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1479\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block1_relu1&#45;&gt;layer4_block0_bn1 -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer3_block1_relu1&#45;&gt;layer4_block0_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1412.12,-47C1421.34,-47 1431.89,-47 1441.8,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1441.82,-50.5001 1451.82,-47 1441.82,-43.5001 1441.82,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1 -->\n",
       "<g id=\"node17\" class=\"node\"><title>layer4_block0_relu1</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1584,-65C1584,-65 1554,-65 1554,-65 1548,-65 1542,-59 1542,-53 1542,-53 1542,-41 1542,-41 1542,-35 1548,-29 1554,-29 1554,-29 1584,-29 1584,-29 1590,-29 1596,-35 1596,-41 1596,-41 1596,-53 1596,-53 1596,-59 1590,-65 1584,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1569\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_bn1&#45;&gt;layer4_block0_relu1 -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>layer4_block0_bn1&#45;&gt;layer4_block0_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1506.4,-47C1514.39,-47 1523.31,-47 1531.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1531.92,-50.5001 1541.92,-47 1531.92,-43.5001 1531.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3 -->\n",
       "<g id=\"node18\" class=\"node\"><title>layer4_block0_conv3</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1674,-65C1674,-65 1644,-65 1644,-65 1638,-65 1632,-59 1632,-53 1632,-53 1632,-41 1632,-41 1632,-35 1638,-29 1644,-29 1644,-29 1674,-29 1674,-29 1680,-29 1686,-35 1686,-41 1686,-41 1686,-53 1686,-53 1686,-59 1680,-65 1674,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1659\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv3</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu1&#45;&gt;layer4_block0_conv3 -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>layer4_block0_relu1&#45;&gt;layer4_block0_conv3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1596.4,-47C1604.39,-47 1613.31,-47 1621.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1621.92,-50.5001 1631.92,-47 1621.92,-43.5001 1621.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1 -->\n",
       "<g id=\"node19\" class=\"node\"><title>layer4_block1_bn1</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1764,-65C1764,-65 1734,-65 1734,-65 1728,-65 1722,-59 1722,-53 1722,-53 1722,-41 1722,-41 1722,-35 1728,-29 1734,-29 1734,-29 1764,-29 1764,-29 1770,-29 1776,-35 1776,-41 1776,-41 1776,-53 1776,-53 1776,-59 1770,-65 1764,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1749\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_conv3&#45;&gt;layer4_block1_bn1 -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>layer4_block0_conv3&#45;&gt;layer4_block1_bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1686.4,-47C1694.39,-47 1703.31,-47 1711.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1711.92,-50.5001 1721.92,-47 1711.92,-43.5001 1711.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1 -->\n",
       "<g id=\"node20\" class=\"node\"><title>layer4_block1_relu1</title>\n",
       "<g id=\"a_node20\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1854,-65C1854,-65 1824,-65 1824,-65 1818,-65 1812,-59 1812,-53 1812,-53 1812,-41 1812,-41 1812,-35 1818,-29 1824,-29 1824,-29 1854,-29 1854,-29 1860,-29 1866,-35 1866,-41 1866,-41 1866,-53 1866,-53 1866,-59 1860,-65 1854,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1839\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu1</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_bn1&#45;&gt;layer4_block1_relu1 -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>layer4_block1_bn1&#45;&gt;layer4_block1_relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1776.4,-47C1784.39,-47 1793.31,-47 1801.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1801.92,-50.5001 1811.92,-47 1801.92,-43.5001 1801.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- final_in -->\n",
       "<g id=\"node21\" class=\"node\"><title>final_in</title>\n",
       "<g id=\"a_node21\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M1944,-65C1944,-65 1914,-65 1914,-65 1908,-65 1902,-59 1902,-53 1902,-53 1902,-41 1902,-41 1902,-35 1908,-29 1914,-29 1914,-29 1944,-29 1944,-29 1950,-29 1956,-35 1956,-41 1956,-41 1956,-53 1956,-53 1956,-59 1950,-65 1944,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1929\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">in</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block1_relu1&#45;&gt;final_in -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>layer4_block1_relu1&#45;&gt;final_in</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1866.4,-47C1874.39,-47 1883.31,-47 1891.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1891.92,-50.5001 1901.92,-47 1891.92,-43.5001 1891.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- final_maxpool -->\n",
       "<g id=\"node22\" class=\"node\"><title>final_maxpool</title>\n",
       "<g id=\"a_node22\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#fdb462\" stroke=\"black\" d=\"M2045,-52C2045,-52 2004,-52 2004,-52 1998,-52 1992,-46 1992,-40 1992,-40 1992,-28 1992,-28 1992,-22 1998,-16 2004,-16 2004,-16 2045,-16 2045,-16 2051,-16 2057,-22 2057,-28 2057,-28 2057,-40 2057,-40 2057,-46 2051,-52 2045,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"2024.5\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">maxpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_maxpool -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>final_in&#45;&gt;final_maxpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1956.04,-43.3788C1964.05,-42.2664 1973.06,-41.0131 1981.83,-39.7933\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1982.52,-43.2312 1991.94,-38.3873 1981.56,-36.2979 1982.52,-43.2312\"/>\n",
       "</g>\n",
       "<!-- final_avgpool -->\n",
       "<g id=\"node23\" class=\"node\"><title>final_avgpool</title>\n",
       "<g id=\"a_node23\"><a xlink:title=\"AvgPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;ceil_mode&#39;: False, &#39;count_include_pad&#39;: True}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M2042.5,-106C2042.5,-106 2006.5,-106 2006.5,-106 2000.5,-106 1994.5,-100 1994.5,-94 1994.5,-94 1994.5,-82 1994.5,-82 1994.5,-76 2000.5,-70 2006.5,-70 2006.5,-70 2042.5,-70 2042.5,-70 2048.5,-70 2054.5,-76 2054.5,-82 2054.5,-82 2054.5,-94 2054.5,-94 2054.5,-100 2048.5,-106 2042.5,-106\"/>\n",
       "<text text-anchor=\"middle\" x=\"2024.5\" y=\"-84.3\" font-family=\"Times,serif\" font-size=\"14.00\">avgpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_in&#45;&gt;final_avgpool -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>final_in&#45;&gt;final_avgpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1956.04,-58.4208C1965.05,-62.3711 1975.35,-66.8851 1985.13,-71.1768\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1983.74,-74.3872 1994.3,-75.1977 1986.55,-67.9764 1983.74,-74.3872\"/>\n",
       "</g>\n",
       "<!-- final_concat -->\n",
       "<g id=\"node24\" class=\"node\"><title>final_concat</title>\n",
       "<g id=\"a_node24\"><a xlink:title=\"Concat {}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M2135,-79C2135,-79 2105,-79 2105,-79 2099,-79 2093,-73 2093,-67 2093,-67 2093,-55 2093,-55 2093,-49 2099,-43 2105,-43 2105,-43 2135,-43 2135,-43 2141,-43 2147,-49 2147,-55 2147,-55 2147,-67 2147,-67 2147,-73 2141,-79 2135,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"2120\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">concat</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_maxpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>final_maxpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2057.15,-43.141C2065.53,-45.5601 2074.63,-48.1884 2083.21,-50.6659\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2082.39,-54.0717 2092.97,-53.4835 2084.33,-47.3465 2082.39,-54.0717\"/>\n",
       "</g>\n",
       "<!-- final_avgpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>final_avgpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2054.57,-79.605C2063.55,-77.0131 2073.54,-74.1266 2082.93,-71.4155\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2084.17,-74.7018 2092.8,-68.5648 2082.22,-67.9766 2084.17,-74.7018\"/>\n",
       "</g>\n",
       "<!-- final_flatten -->\n",
       "<g id=\"node25\" class=\"node\"><title>final_flatten</title>\n",
       "<g id=\"a_node25\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M2225,-79C2225,-79 2195,-79 2195,-79 2189,-79 2183,-73 2183,-67 2183,-67 2183,-55 2183,-55 2183,-49 2189,-43 2195,-43 2195,-43 2225,-43 2225,-43 2231,-43 2237,-49 2237,-55 2237,-55 2237,-67 2237,-67 2237,-73 2231,-79 2225,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"2210\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_concat&#45;&gt;final_flatten -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>final_concat&#45;&gt;final_flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2147.4,-61C2155.39,-61 2164.31,-61 2172.82,-61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2172.92,-64.5001 2182.92,-61 2172.92,-57.5001 2172.92,-64.5001\"/>\n",
       "</g>\n",
       "<!-- final_linear -->\n",
       "<g id=\"node26\" class=\"node\"><title>final_linear</title>\n",
       "<g id=\"a_node26\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: Parameter containing:\\ntensor([ 0.0190, &#45;0.0074, &#160;0.0270, &#45;0.0347, &#45;0.0215, &#160;0.0318, &#160;0.0199, &#45;0.0423,\\n &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#45;0.0143, &#160;0.0422], requires_grad=True)}\">\n",
       "<path fill=\"#ccebc5\" stroke=\"black\" d=\"M2315,-79C2315,-79 2285,-79 2285,-79 2279,-79 2273,-73 2273,-67 2273,-67 2273,-55 2273,-55 2273,-49 2279,-43 2285,-43 2285,-43 2315,-43 2315,-43 2321,-43 2327,-49 2327,-55 2327,-55 2327,-67 2327,-67 2327,-73 2321,-79 2315,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"2300\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_flatten&#45;&gt;final_linear -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>final_flatten&#45;&gt;final_linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2237.4,-61C2245.39,-61 2254.31,-61 2262.82,-61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2262.92,-64.5001 2272.92,-61 2262.92,-57.5001 2262.92,-64.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node27\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node27\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M2416,-79C2416,-79 2375,-79 2375,-79 2369,-79 2363,-73 2363,-67 2363,-67 2363,-55 2363,-55 2363,-49 2369,-43 2375,-43 2375,-43 2416,-43 2416,-43 2422,-43 2428,-49 2428,-55 2428,-55 2428,-67 2428,-67 2428,-73 2422,-79 2416,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"2395.5\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_linear&#45;&gt;classifier -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>final_linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2327.04,-61C2335.05,-61 2344.06,-61 2352.83,-61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2352.95,-64.5001 2362.94,-61 2352.94,-57.5001 2352.95,-64.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node28\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-65C42,-65 12,-65 12,-65 6,-65 0,-59 0,-53 0,-53 0,-41 0,-41 0,-35 6,-29 12,-29 12,-29 42,-29 42,-29 48,-29 54,-35 54,-41 54,-41 54,-53 54,-53 54,-59 48,-65 42,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-47C62.3932,-47 71.3106,-47 79.8241,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-50.5001 89.919,-47 79.919,-43.5001 79.919,-50.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7f0103eafb70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.1000       1.8775       1.9952       0.2584       0.1548       1.7645       0.3561       2.0323\n",
      "           2       0.2000       1.8689       1.7656       0.3524       0.1450       1.6898       0.3839       4.0462\n",
      "           3       0.3000       1.8307       1.6943       0.3821       0.1550       1.6774       0.3876       6.0319\n",
      "           4       0.4000       1.8995       1.6663       0.3929       0.1440       1.5992       0.4225       8.0755\n",
      "           5       0.3750       1.7525       1.6494       0.4032       0.1564       1.5825       0.4284       9.9843\n",
      "           6       0.3500       1.8375       1.6126       0.4125       0.1465       1.5598       0.4356      11.9684\n",
      "           7       0.3250       1.7787       1.5944       0.4254       0.1567       1.6138       0.4260      13.9037\n",
      "           8       0.3000       1.9086       1.5611       0.4361       0.1656       1.5451       0.4296      15.9778\n",
      "           9       0.2750       1.9406       1.5505       0.4402       0.1557       1.6577       0.4103      18.0742\n",
      "          10       0.2500       1.9403       1.5371       0.4456       0.1571       1.5806       0.4316      20.1716\n",
      "          11       0.2250       1.8400       1.5152       0.4545       0.1459       1.4805       0.4649      22.1575\n",
      "          12       0.2000       1.6703       1.4903       0.4653       0.1537       1.5323       0.4492      23.9815\n",
      "          13       0.1750       1.8281       1.4755       0.4681       0.1455       1.4409       0.4771      25.9551\n",
      "          14       0.1500       1.7502       1.4549       0.4754       0.1534       1.3836       0.5074      27.8587\n",
      "          15       0.1250       1.8461       1.4320       0.4892       0.1543       1.3549       0.5181      29.8591\n",
      "          16       0.1000       1.8519       1.3987       0.4977       0.1556       1.4099       0.4980      31.8666\n",
      "          17       0.0750       1.9110       1.3849       0.5016       0.1560       1.3574       0.5101      33.9335\n",
      "          18       0.0500       1.7852       1.3605       0.5154       0.1569       1.2988       0.5406      35.8757\n",
      "          19       0.0250       1.8492       1.3325       0.5261       0.1562       1.2771       0.5433      37.8810\n",
      "          20       0.0000       1.8251       1.3041       0.5361       0.1545       1.2417       0.5541      39.8607\n"
     ]
    }
   ],
   "source": [
    "def shortcut_block(c_in, c_out, stride, **kw):\n",
    "    block = {\n",
    "        'bn1': batch_norm(c_in, **kw),\n",
    "        'relu1': nn.ReLU(True),\n",
    "    }\n",
    "    projection = (stride != 1) or (c_in != c_out)    \n",
    "    if projection:\n",
    "        block['conv3'] = (nn.Conv2d(c_in, c_out, kernel_size=1, stride=stride, padding=0, bias=False), ['relu1'])\n",
    "    return block\n",
    "\n",
    "lr_schedule = PiecewiseLinear([0, 4, 20], [0, 0.4, 0])\n",
    "batch_size = 512\n",
    "\n",
    "n = DAWN_net(block=shortcut_block)\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(8,8)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 4: Architecture](https://www.myrtle.ai/2018/10/26/how_to_train_your_resnet_4/)  - backbone, remove repeat bn-relu (32s; test acc 56.0%)\n",
    "\n",
    "Removing the repeated batch norm-ReLU groups, reduces training time to 32s and leaves test accuracy approximately unchanged."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"108pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 107.51\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.702668 0.702668) rotate(0) translate(4 149)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-149 1533,-149 1533,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-21C94,-21 320,-21 320,-21 326,-21 332,-27 332,-33 332,-33 332,-84 332,-84 332,-90 326,-96 320,-96 320,-96 94,-96 94,-96 88,-96 82,-90 82,-84 82,-84 82,-33 82,-33 82,-27 88,-21 94,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M356,-13C356,-13 598,-13 598,-13 604,-13 610,-19 610,-25 610,-25 610,-115 610,-115 610,-121 604,-127 598,-127 598,-127 356,-127 356,-127 350,-127 344,-121 344,-115 344,-115 344,-25 344,-25 344,-19 350,-13 356,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer2_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-21C364,-21 590,-21 590,-21 596,-21 602,-27 602,-33 602,-33 602,-84 602,-84 602,-90 596,-96 590,-96 590,-96 364,-96 364,-96 358,-96 352,-90 352,-84 352,-84 352,-33 352,-33 352,-27 358,-21 364,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M630,-13C630,-13 872,-13 872,-13 878,-13 884,-19 884,-25 884,-25 884,-115 884,-115 884,-121 878,-127 872,-127 872,-127 630,-127 630,-127 624,-127 618,-121 618,-115 618,-115 618,-25 618,-25 618,-19 624,-13 630,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"751\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer3_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M638,-21C638,-21 864,-21 864,-21 870,-21 876,-27 876,-33 876,-33 876,-84 876,-84 876,-90 870,-96 864,-96 864,-96 638,-96 638,-96 632,-96 626,-90 626,-84 626,-84 626,-33 626,-33 626,-27 632,-21 638,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"751\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer4</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M904,-13C904,-13 1146,-13 1146,-13 1152,-13 1158,-19 1158,-25 1158,-25 1158,-115 1158,-115 1158,-121 1152,-127 1146,-127 1146,-127 904,-127 904,-127 898,-127 892,-121 892,-115 892,-115 892,-25 892,-25 892,-19 898,-13 904,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"1025\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer4</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer4_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M912,-21C912,-21 1138,-21 1138,-21 1144,-21 1150,-27 1150,-33 1150,-33 1150,-84 1150,-84 1150,-90 1144,-96 1138,-96 1138,-96 912,-96 912,-96 906,-96 900,-90 900,-84 900,-84 900,-33 900,-33 900,-27 906,-21 912,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"1025\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_final</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1182,-8C1182,-8 1509,-8 1509,-8 1515,-8 1521,-14 1521,-20 1521,-20 1521,-125 1521,-125 1521,-131 1515,-137 1509,-137 1509,-137 1182,-137 1182,-137 1176,-137 1170,-131 1170,-125 1170,-125 1170,-20 1170,-20 1170,-14 1176,-8 1182,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"1345.5\" y=\"-121.8\" font-family=\"Times,serif\" font-size=\"14.00\">final</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-65C132,-65 102,-65 102,-65 96,-65 90,-59 90,-53 90,-53 90,-41 90,-41 90,-35 96,-29 102,-29 102,-29 132,-29 132,-29 138,-29 144,-35 144,-41 144,-41 144,-53 144,-53 144,-59 138,-65 132,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn -->\n",
       "<g id=\"node2\" class=\"node\"><title>prep_bn</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-65C222,-65 192,-65 192,-65 186,-65 180,-59 180,-53 180,-53 180,-41 180,-41 180,-35 186,-29 192,-29 192,-29 222,-29 222,-29 228,-29 234,-35 234,-41 234,-41 234,-53 234,-53 234,-59 228,-65 222,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;prep_bn -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;prep_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-47C152.393,-47 161.311,-47 169.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-50.5001 179.919,-47 169.919,-43.5001 169.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- prep_relu -->\n",
       "<g id=\"node3\" class=\"node\"><title>prep_relu</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-65C312,-65 282,-65 282,-65 276,-65 270,-59 270,-53 270,-53 270,-41 270,-41 270,-35 276,-29 282,-29 282,-29 312,-29 312,-29 318,-29 324,-35 324,-41 324,-41 324,-53 324,-53 324,-59 318,-65 312,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn&#45;&gt;prep_relu -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>prep_bn&#45;&gt;prep_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-47C242.393,-47 251.311,-47 259.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-50.5001 269.919,-47 259.919,-43.5001 259.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer2_block0_conv</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-65C402,-65 372,-65 372,-65 366,-65 360,-59 360,-53 360,-53 360,-41 360,-41 360,-35 366,-29 372,-29 372,-29 402,-29 402,-29 408,-29 414,-35 414,-41 414,-41 414,-53 414,-53 414,-59 408,-65 402,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_relu&#45;&gt;layer2_block0_conv -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>prep_relu&#45;&gt;layer2_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-47C332.393,-47 341.311,-47 349.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.919,-50.5001 359.919,-47 349.919,-43.5001 349.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_bn -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer2_block0_bn</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-65C492,-65 462,-65 462,-65 456,-65 450,-59 450,-53 450,-53 450,-41 450,-41 450,-35 456,-29 462,-29 462,-29 492,-29 492,-29 498,-29 504,-35 504,-41 504,-41 504,-53 504,-53 504,-59 498,-65 492,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_conv&#45;&gt;layer2_block0_bn -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer2_block0_conv&#45;&gt;layer2_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-47C422.393,-47 431.311,-47 439.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-50.5001 449.919,-47 439.919,-43.5001 439.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_relu -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer2_block0_relu</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-65C582,-65 552,-65 552,-65 546,-65 540,-59 540,-53 540,-53 540,-41 540,-41 540,-35 546,-29 552,-29 552,-29 582,-29 582,-29 588,-29 594,-35 594,-41 594,-41 594,-53 594,-53 594,-59 588,-65 582,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_bn&#45;&gt;layer2_block0_relu -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer2_block0_bn&#45;&gt;layer2_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-47C512.393,-47 521.311,-47 529.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-50.5001 539.919,-47 529.919,-43.5001 529.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer3_block0_conv</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M676,-65C676,-65 646,-65 646,-65 640,-65 634,-59 634,-53 634,-53 634,-41 634,-41 634,-35 640,-29 646,-29 646,-29 676,-29 676,-29 682,-29 688,-35 688,-41 688,-41 688,-53 688,-53 688,-59 682,-65 676,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"661\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu&#45;&gt;layer3_block0_conv -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer2_block0_relu&#45;&gt;layer3_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.117,-47C603.343,-47 613.891,-47 623.802,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"623.817,-50.5001 633.817,-47 623.817,-43.5001 623.817,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_bn -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer3_block0_bn</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M766,-65C766,-65 736,-65 736,-65 730,-65 724,-59 724,-53 724,-53 724,-41 724,-41 724,-35 730,-29 736,-29 736,-29 766,-29 766,-29 772,-29 778,-35 778,-41 778,-41 778,-53 778,-53 778,-59 772,-65 766,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"751\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_conv&#45;&gt;layer3_block0_bn -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer3_block0_conv&#45;&gt;layer3_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M688.403,-47C696.393,-47 705.311,-47 713.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"713.919,-50.5001 723.919,-47 713.919,-43.5001 713.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_relu -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer3_block0_relu</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M856,-65C856,-65 826,-65 826,-65 820,-65 814,-59 814,-53 814,-53 814,-41 814,-41 814,-35 820,-29 826,-29 826,-29 856,-29 856,-29 862,-29 868,-35 868,-41 868,-41 868,-53 868,-53 868,-59 862,-65 856,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"841\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_bn&#45;&gt;layer3_block0_relu -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer3_block0_bn&#45;&gt;layer3_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M778.403,-47C786.393,-47 795.311,-47 803.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"803.919,-50.5001 813.919,-47 803.919,-43.5001 803.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer4_block0_conv</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (1, 1), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (0, 0), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M950,-65C950,-65 920,-65 920,-65 914,-65 908,-59 908,-53 908,-53 908,-41 908,-41 908,-35 914,-29 920,-29 920,-29 950,-29 950,-29 956,-29 962,-35 962,-41 962,-41 962,-53 962,-53 962,-59 956,-65 950,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"935\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu&#45;&gt;layer4_block0_conv -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer3_block0_relu&#45;&gt;layer4_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M868.117,-47C877.343,-47 887.891,-47 897.802,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"897.817,-50.5001 907.817,-47 897.817,-43.5001 897.817,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_bn -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer4_block0_bn</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1040,-65C1040,-65 1010,-65 1010,-65 1004,-65 998,-59 998,-53 998,-53 998,-41 998,-41 998,-35 1004,-29 1010,-29 1010,-29 1040,-29 1040,-29 1046,-29 1052,-35 1052,-41 1052,-41 1052,-53 1052,-53 1052,-59 1046,-65 1040,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1025\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_conv&#45;&gt;layer4_block0_bn -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer4_block0_conv&#45;&gt;layer4_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M962.403,-47C970.393,-47 979.311,-47 987.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"987.919,-50.5001 997.919,-47 987.919,-43.5001 987.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_relu -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer4_block0_relu</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1130,-65C1130,-65 1100,-65 1100,-65 1094,-65 1088,-59 1088,-53 1088,-53 1088,-41 1088,-41 1088,-35 1094,-29 1100,-29 1100,-29 1130,-29 1130,-29 1136,-29 1142,-35 1142,-41 1142,-41 1142,-53 1142,-53 1142,-59 1136,-65 1130,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1115\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_bn&#45;&gt;layer4_block0_relu -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer4_block0_bn&#45;&gt;layer4_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1052.4,-47C1060.39,-47 1069.31,-47 1077.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1077.92,-50.5001 1087.92,-47 1077.92,-43.5001 1077.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- final_maxpool -->\n",
       "<g id=\"node13\" class=\"node\"><title>final_maxpool</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1231,-106C1231,-106 1190,-106 1190,-106 1184,-106 1178,-100 1178,-94 1178,-94 1178,-82 1178,-82 1178,-76 1184,-70 1190,-70 1190,-70 1231,-70 1231,-70 1237,-70 1243,-76 1243,-82 1243,-82 1243,-94 1243,-94 1243,-100 1237,-106 1231,-106\"/>\n",
       "<text text-anchor=\"middle\" x=\"1210.5\" y=\"-84.3\" font-family=\"Times,serif\" font-size=\"14.00\">maxpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu&#45;&gt;final_maxpool -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer4_block0_relu&#45;&gt;final_maxpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1142.04,-58.4208C1150.31,-62.0434 1159.65,-66.1401 1168.69,-70.1047\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1167.38,-73.3525 1177.94,-74.163 1170.19,-66.9417 1167.38,-73.3525\"/>\n",
       "</g>\n",
       "<!-- final_avgpool -->\n",
       "<g id=\"node14\" class=\"node\"><title>final_avgpool</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"AvgPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;ceil_mode&#39;: False, &#39;count_include_pad&#39;: True}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M1228.5,-52C1228.5,-52 1192.5,-52 1192.5,-52 1186.5,-52 1180.5,-46 1180.5,-40 1180.5,-40 1180.5,-28 1180.5,-28 1180.5,-22 1186.5,-16 1192.5,-16 1192.5,-16 1228.5,-16 1228.5,-16 1234.5,-16 1240.5,-22 1240.5,-28 1240.5,-28 1240.5,-40 1240.5,-40 1240.5,-46 1234.5,-52 1228.5,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1210.5\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">avgpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu&#45;&gt;final_avgpool -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer4_block0_relu&#45;&gt;final_avgpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1142.04,-43.3788C1150.77,-42.165 1160.71,-40.7833 1170.23,-39.4606\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1170.88,-42.9031 1180.3,-38.0593 1169.92,-35.9698 1170.88,-42.9031\"/>\n",
       "</g>\n",
       "<!-- final_concat -->\n",
       "<g id=\"node15\" class=\"node\"><title>final_concat</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"Concat {}\">\n",
       "<path fill=\"#fdb462\" stroke=\"black\" d=\"M1321,-79C1321,-79 1291,-79 1291,-79 1285,-79 1279,-73 1279,-67 1279,-67 1279,-55 1279,-55 1279,-49 1285,-43 1291,-43 1291,-43 1321,-43 1321,-43 1327,-43 1333,-49 1333,-55 1333,-55 1333,-67 1333,-67 1333,-73 1327,-79 1321,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"1306\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">concat</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_maxpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>final_maxpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1243.15,-78.859C1251.53,-76.4399 1260.63,-73.8116 1269.21,-71.3341\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1270.33,-74.6535 1278.97,-68.5165 1268.39,-67.9283 1270.33,-74.6535\"/>\n",
       "</g>\n",
       "<!-- final_avgpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>final_avgpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1240.57,-42.395C1249.55,-44.9869 1259.54,-47.8734 1268.93,-50.5845\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1268.22,-54.0234 1278.8,-53.4352 1270.17,-47.2982 1268.22,-54.0234\"/>\n",
       "</g>\n",
       "<!-- final_flatten -->\n",
       "<g id=\"node16\" class=\"node\"><title>final_flatten</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M1411,-79C1411,-79 1381,-79 1381,-79 1375,-79 1369,-73 1369,-67 1369,-67 1369,-55 1369,-55 1369,-49 1375,-43 1381,-43 1381,-43 1411,-43 1411,-43 1417,-43 1423,-49 1423,-55 1423,-55 1423,-67 1423,-67 1423,-73 1417,-79 1411,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"1396\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_concat&#45;&gt;final_flatten -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>final_concat&#45;&gt;final_flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1333.4,-61C1341.39,-61 1350.31,-61 1358.82,-61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1358.92,-64.5001 1368.92,-61 1358.92,-57.5001 1358.92,-64.5001\"/>\n",
       "</g>\n",
       "<!-- final_linear -->\n",
       "<g id=\"node17\" class=\"node\"><title>final_linear</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: Parameter containing:\\ntensor([&#45;0.0082, &#160;0.0309, &#160;0.0371, &#160;0.0122, &#45;0.0077, &#45;0.0393, &#160;0.0263, &#160;0.0084,\\n &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#45;0.0174, &#160;0.0185], requires_grad=True)}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M1501,-79C1501,-79 1471,-79 1471,-79 1465,-79 1459,-73 1459,-67 1459,-67 1459,-55 1459,-55 1459,-49 1465,-43 1471,-43 1471,-43 1501,-43 1501,-43 1507,-43 1513,-49 1513,-55 1513,-55 1513,-67 1513,-67 1513,-73 1507,-79 1501,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"1486\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_flatten&#45;&gt;final_linear -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>final_flatten&#45;&gt;final_linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1423.4,-61C1431.39,-61 1440.31,-61 1448.82,-61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1448.92,-64.5001 1458.92,-61 1448.92,-57.5001 1448.92,-64.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node18\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-65C42,-65 12,-65 12,-65 6,-65 0,-59 0,-53 0,-53 0,-41 0,-41 0,-35 6,-29 12,-29 12,-29 42,-29 42,-29 48,-29 54,-35 54,-41 54,-41 54,-53 54,-53 54,-59 48,-65 42,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-47C62.3932,-47 71.3106,-47 79.8241,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-50.5001 89.919,-47 79.919,-43.5001 79.919,-50.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7fcd2b22d358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.1000       2.0275       1.9549       0.2788       0.1974       1.7310       0.3777       2.2249\n",
      "           2       0.2000       1.6905       1.7492       0.3624       0.1465       1.6506       0.3997       4.0619\n",
      "           3       0.3000       1.6741       1.6697       0.3913       0.1418       1.6870       0.3864       5.8779\n",
      "           4       0.4000       1.7245       1.6648       0.3987       0.1441       1.6405       0.4068       7.7464\n",
      "           5       0.3750       1.5438       1.6470       0.4054       0.1373       1.5562       0.4478       9.4275\n",
      "           6       0.3500       1.5782       1.6121       0.4199       0.1462       1.5877       0.4286      11.1519\n",
      "           7       0.3250       1.6797       1.5869       0.4275       0.1476       1.5179       0.4499      12.9793\n",
      "           8       0.3000       1.7134       1.5592       0.4379       0.1378       1.5115       0.4574      14.8305\n",
      "           9       0.2750       1.6557       1.5419       0.4447       0.1452       1.4563       0.4772      16.6314\n",
      "          10       0.2500       1.6759       1.5269       0.4503       0.1445       1.5220       0.4514      18.4518\n",
      "          11       0.2250       1.6124       1.5006       0.4597       0.1465       1.4879       0.4648      20.2108\n",
      "          12       0.2000       1.7304       1.4856       0.4668       0.1432       1.4202       0.4926      22.0843\n",
      "          13       0.1750       1.6290       1.4676       0.4729       0.1379       1.4023       0.4977      23.8512\n",
      "          14       0.1500       1.5444       1.4424       0.4806       0.1443       1.4114       0.4913      25.5399\n",
      "          15       0.1250       1.5514       1.4236       0.4893       0.1517       1.3544       0.5233      27.2430\n",
      "          16       0.1000       1.7442       1.3950       0.5032       0.1461       1.3603       0.5165      29.1333\n",
      "          17       0.0750       1.6924       1.3745       0.5086       0.1495       1.3167       0.5257      30.9752\n",
      "          18       0.0500       1.6672       1.3573       0.5129       0.1451       1.3070       0.5371      32.7874\n",
      "          19       0.0250       1.6698       1.3314       0.5247       0.1468       1.2594       0.5536      34.6040\n",
      "          20       0.0000       1.7767       1.2969       0.5369       0.1464       1.2386       0.5599      36.5271\n"
     ]
    }
   ],
   "source": [
    "def shortcut_block(c_in, c_out, stride, **kw):\n",
    "    projection = (stride != 1) or (c_in != c_out)\n",
    "    if projection:\n",
    "        return {\n",
    "            'conv':  nn.Conv2d(c_in, c_out, kernel_size=1, stride=stride, padding=0, bias=False), \n",
    "            'bn': batch_norm(c_out, **kw),\n",
    "            'relu': nn.ReLU(True),\n",
    "        }\n",
    "    else:\n",
    "        return {'id': Identity()}\n",
    "\n",
    "lr_schedule = PiecewiseLinear([0, 4, 20], [0, 0.4, 0])\n",
    "batch_size = 512\n",
    "\n",
    "n = DAWN_net(block=shortcut_block, prep_bn_relu=True)\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(8,8)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 4: Architecture](https://www.myrtle.ai/2018/10/26/how_to_train_your_resnet_4/)  - backbone, 3x3 convs (36s; test acc 85.6%)\n",
    "\n",
    "A serious shortcoming of this network is that the downsampling convolutions have 1x1 kernels and a stride of two, so that rather than enlarging the receptive field they are simply discarding information. \n",
    "\n",
    "If we replace these with 3x3 convolutions, things improve considerably and test accuracy after 20 epochs is 85.6% in a time of 36s."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"108pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 107.51\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.702668 0.702668) rotate(0) translate(4 149)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-149 1533,-149 1533,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-21C94,-21 320,-21 320,-21 326,-21 332,-27 332,-33 332,-33 332,-84 332,-84 332,-90 326,-96 320,-96 320,-96 94,-96 94,-96 88,-96 82,-90 82,-84 82,-84 82,-33 82,-33 82,-27 88,-21 94,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M356,-13C356,-13 598,-13 598,-13 604,-13 610,-19 610,-25 610,-25 610,-115 610,-115 610,-121 604,-127 598,-127 598,-127 356,-127 356,-127 350,-127 344,-121 344,-115 344,-115 344,-25 344,-25 344,-19 350,-13 356,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer2_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-21C364,-21 590,-21 590,-21 596,-21 602,-27 602,-33 602,-33 602,-84 602,-84 602,-90 596,-96 590,-96 590,-96 364,-96 364,-96 358,-96 352,-90 352,-84 352,-84 352,-33 352,-33 352,-27 358,-21 364,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M630,-13C630,-13 872,-13 872,-13 878,-13 884,-19 884,-25 884,-25 884,-115 884,-115 884,-121 878,-127 872,-127 872,-127 630,-127 630,-127 624,-127 618,-121 618,-115 618,-115 618,-25 618,-25 618,-19 624,-13 630,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"751\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer3_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M638,-21C638,-21 864,-21 864,-21 870,-21 876,-27 876,-33 876,-33 876,-84 876,-84 876,-90 870,-96 864,-96 864,-96 638,-96 638,-96 632,-96 626,-90 626,-84 626,-84 626,-33 626,-33 626,-27 632,-21 638,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"751\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer4</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M904,-13C904,-13 1146,-13 1146,-13 1152,-13 1158,-19 1158,-25 1158,-25 1158,-115 1158,-115 1158,-121 1152,-127 1146,-127 1146,-127 904,-127 904,-127 898,-127 892,-121 892,-115 892,-115 892,-25 892,-25 892,-19 898,-13 904,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"1025\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer4</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer4_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M912,-21C912,-21 1138,-21 1138,-21 1144,-21 1150,-27 1150,-33 1150,-33 1150,-84 1150,-84 1150,-90 1144,-96 1138,-96 1138,-96 912,-96 912,-96 906,-96 900,-90 900,-84 900,-84 900,-33 900,-33 900,-27 906,-21 912,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"1025\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_final</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1182,-8C1182,-8 1509,-8 1509,-8 1515,-8 1521,-14 1521,-20 1521,-20 1521,-125 1521,-125 1521,-131 1515,-137 1509,-137 1509,-137 1182,-137 1182,-137 1176,-137 1170,-131 1170,-125 1170,-125 1170,-20 1170,-20 1170,-14 1176,-8 1182,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"1345.5\" y=\"-121.8\" font-family=\"Times,serif\" font-size=\"14.00\">final</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-65C132,-65 102,-65 102,-65 96,-65 90,-59 90,-53 90,-53 90,-41 90,-41 90,-35 96,-29 102,-29 102,-29 132,-29 132,-29 138,-29 144,-35 144,-41 144,-41 144,-53 144,-53 144,-59 138,-65 132,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn -->\n",
       "<g id=\"node2\" class=\"node\"><title>prep_bn</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-65C222,-65 192,-65 192,-65 186,-65 180,-59 180,-53 180,-53 180,-41 180,-41 180,-35 186,-29 192,-29 192,-29 222,-29 222,-29 228,-29 234,-35 234,-41 234,-41 234,-53 234,-53 234,-59 228,-65 222,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;prep_bn -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;prep_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-47C152.393,-47 161.311,-47 169.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-50.5001 179.919,-47 169.919,-43.5001 169.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- prep_relu -->\n",
       "<g id=\"node3\" class=\"node\"><title>prep_relu</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-65C312,-65 282,-65 282,-65 276,-65 270,-59 270,-53 270,-53 270,-41 270,-41 270,-35 276,-29 282,-29 282,-29 312,-29 312,-29 318,-29 324,-35 324,-41 324,-41 324,-53 324,-53 324,-59 318,-65 312,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn&#45;&gt;prep_relu -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>prep_bn&#45;&gt;prep_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-47C242.393,-47 251.311,-47 259.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-50.5001 269.919,-47 259.919,-43.5001 259.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer2_block0_conv</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-65C402,-65 372,-65 372,-65 366,-65 360,-59 360,-53 360,-53 360,-41 360,-41 360,-35 366,-29 372,-29 372,-29 402,-29 402,-29 408,-29 414,-35 414,-41 414,-41 414,-53 414,-53 414,-59 408,-65 402,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_relu&#45;&gt;layer2_block0_conv -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>prep_relu&#45;&gt;layer2_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-47C332.393,-47 341.311,-47 349.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.919,-50.5001 359.919,-47 349.919,-43.5001 349.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_bn -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer2_block0_bn</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-65C492,-65 462,-65 462,-65 456,-65 450,-59 450,-53 450,-53 450,-41 450,-41 450,-35 456,-29 462,-29 462,-29 492,-29 492,-29 498,-29 504,-35 504,-41 504,-41 504,-53 504,-53 504,-59 498,-65 492,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_conv&#45;&gt;layer2_block0_bn -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer2_block0_conv&#45;&gt;layer2_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-47C422.393,-47 431.311,-47 439.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-50.5001 449.919,-47 439.919,-43.5001 439.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_relu -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer2_block0_relu</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-65C582,-65 552,-65 552,-65 546,-65 540,-59 540,-53 540,-53 540,-41 540,-41 540,-35 546,-29 552,-29 552,-29 582,-29 582,-29 588,-29 594,-35 594,-41 594,-41 594,-53 594,-53 594,-59 588,-65 582,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_bn&#45;&gt;layer2_block0_relu -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer2_block0_bn&#45;&gt;layer2_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-47C512.393,-47 521.311,-47 529.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-50.5001 539.919,-47 529.919,-43.5001 529.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer3_block0_conv</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M676,-65C676,-65 646,-65 646,-65 640,-65 634,-59 634,-53 634,-53 634,-41 634,-41 634,-35 640,-29 646,-29 646,-29 676,-29 676,-29 682,-29 688,-35 688,-41 688,-41 688,-53 688,-53 688,-59 682,-65 676,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"661\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu&#45;&gt;layer3_block0_conv -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer2_block0_relu&#45;&gt;layer3_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.117,-47C603.343,-47 613.891,-47 623.802,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"623.817,-50.5001 633.817,-47 623.817,-43.5001 623.817,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_bn -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer3_block0_bn</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M766,-65C766,-65 736,-65 736,-65 730,-65 724,-59 724,-53 724,-53 724,-41 724,-41 724,-35 730,-29 736,-29 736,-29 766,-29 766,-29 772,-29 778,-35 778,-41 778,-41 778,-53 778,-53 778,-59 772,-65 766,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"751\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_conv&#45;&gt;layer3_block0_bn -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer3_block0_conv&#45;&gt;layer3_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M688.403,-47C696.393,-47 705.311,-47 713.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"713.919,-50.5001 723.919,-47 713.919,-43.5001 713.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_relu -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer3_block0_relu</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M856,-65C856,-65 826,-65 826,-65 820,-65 814,-59 814,-53 814,-53 814,-41 814,-41 814,-35 820,-29 826,-29 826,-29 856,-29 856,-29 862,-29 868,-35 868,-41 868,-41 868,-53 868,-53 868,-59 862,-65 856,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"841\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_bn&#45;&gt;layer3_block0_relu -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer3_block0_bn&#45;&gt;layer3_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M778.403,-47C786.393,-47 795.311,-47 803.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"803.919,-50.5001 813.919,-47 803.919,-43.5001 803.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer4_block0_conv</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (2, 2), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M950,-65C950,-65 920,-65 920,-65 914,-65 908,-59 908,-53 908,-53 908,-41 908,-41 908,-35 914,-29 920,-29 920,-29 950,-29 950,-29 956,-29 962,-35 962,-41 962,-41 962,-53 962,-53 962,-59 956,-65 950,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"935\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu&#45;&gt;layer4_block0_conv -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer3_block0_relu&#45;&gt;layer4_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M868.117,-47C877.343,-47 887.891,-47 897.802,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"897.817,-50.5001 907.817,-47 897.817,-43.5001 897.817,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_bn -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer4_block0_bn</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1040,-65C1040,-65 1010,-65 1010,-65 1004,-65 998,-59 998,-53 998,-53 998,-41 998,-41 998,-35 1004,-29 1010,-29 1010,-29 1040,-29 1040,-29 1046,-29 1052,-35 1052,-41 1052,-41 1052,-53 1052,-53 1052,-59 1046,-65 1040,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1025\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_conv&#45;&gt;layer4_block0_bn -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer4_block0_conv&#45;&gt;layer4_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M962.403,-47C970.393,-47 979.311,-47 987.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"987.919,-50.5001 997.919,-47 987.919,-43.5001 987.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_relu -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer4_block0_relu</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1130,-65C1130,-65 1100,-65 1100,-65 1094,-65 1088,-59 1088,-53 1088,-53 1088,-41 1088,-41 1088,-35 1094,-29 1100,-29 1100,-29 1130,-29 1130,-29 1136,-29 1142,-35 1142,-41 1142,-41 1142,-53 1142,-53 1142,-59 1136,-65 1130,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1115\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_bn&#45;&gt;layer4_block0_relu -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer4_block0_bn&#45;&gt;layer4_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1052.4,-47C1060.39,-47 1069.31,-47 1077.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1077.92,-50.5001 1087.92,-47 1077.92,-43.5001 1077.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- final_maxpool -->\n",
       "<g id=\"node13\" class=\"node\"><title>final_maxpool</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1231,-106C1231,-106 1190,-106 1190,-106 1184,-106 1178,-100 1178,-94 1178,-94 1178,-82 1178,-82 1178,-76 1184,-70 1190,-70 1190,-70 1231,-70 1231,-70 1237,-70 1243,-76 1243,-82 1243,-82 1243,-94 1243,-94 1243,-100 1237,-106 1231,-106\"/>\n",
       "<text text-anchor=\"middle\" x=\"1210.5\" y=\"-84.3\" font-family=\"Times,serif\" font-size=\"14.00\">maxpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu&#45;&gt;final_maxpool -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer4_block0_relu&#45;&gt;final_maxpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1142.04,-58.4208C1150.31,-62.0434 1159.65,-66.1401 1168.69,-70.1047\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1167.38,-73.3525 1177.94,-74.163 1170.19,-66.9417 1167.38,-73.3525\"/>\n",
       "</g>\n",
       "<!-- final_avgpool -->\n",
       "<g id=\"node14\" class=\"node\"><title>final_avgpool</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"AvgPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;ceil_mode&#39;: False, &#39;count_include_pad&#39;: True}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M1228.5,-52C1228.5,-52 1192.5,-52 1192.5,-52 1186.5,-52 1180.5,-46 1180.5,-40 1180.5,-40 1180.5,-28 1180.5,-28 1180.5,-22 1186.5,-16 1192.5,-16 1192.5,-16 1228.5,-16 1228.5,-16 1234.5,-16 1240.5,-22 1240.5,-28 1240.5,-28 1240.5,-40 1240.5,-40 1240.5,-46 1234.5,-52 1228.5,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1210.5\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">avgpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu&#45;&gt;final_avgpool -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer4_block0_relu&#45;&gt;final_avgpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1142.04,-43.3788C1150.77,-42.165 1160.71,-40.7833 1170.23,-39.4606\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1170.88,-42.9031 1180.3,-38.0593 1169.92,-35.9698 1170.88,-42.9031\"/>\n",
       "</g>\n",
       "<!-- final_concat -->\n",
       "<g id=\"node15\" class=\"node\"><title>final_concat</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"Concat {}\">\n",
       "<path fill=\"#fdb462\" stroke=\"black\" d=\"M1321,-79C1321,-79 1291,-79 1291,-79 1285,-79 1279,-73 1279,-67 1279,-67 1279,-55 1279,-55 1279,-49 1285,-43 1291,-43 1291,-43 1321,-43 1321,-43 1327,-43 1333,-49 1333,-55 1333,-55 1333,-67 1333,-67 1333,-73 1327,-79 1321,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"1306\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">concat</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_maxpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>final_maxpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1243.15,-78.859C1251.53,-76.4399 1260.63,-73.8116 1269.21,-71.3341\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1270.33,-74.6535 1278.97,-68.5165 1268.39,-67.9283 1270.33,-74.6535\"/>\n",
       "</g>\n",
       "<!-- final_avgpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>final_avgpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1240.57,-42.395C1249.55,-44.9869 1259.54,-47.8734 1268.93,-50.5845\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1268.22,-54.0234 1278.8,-53.4352 1270.17,-47.2982 1268.22,-54.0234\"/>\n",
       "</g>\n",
       "<!-- final_flatten -->\n",
       "<g id=\"node16\" class=\"node\"><title>final_flatten</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M1411,-79C1411,-79 1381,-79 1381,-79 1375,-79 1369,-73 1369,-67 1369,-67 1369,-55 1369,-55 1369,-49 1375,-43 1381,-43 1381,-43 1411,-43 1411,-43 1417,-43 1423,-49 1423,-55 1423,-55 1423,-67 1423,-67 1423,-73 1417,-79 1411,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"1396\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_concat&#45;&gt;final_flatten -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>final_concat&#45;&gt;final_flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1333.4,-61C1341.39,-61 1350.31,-61 1358.82,-61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1358.92,-64.5001 1368.92,-61 1358.92,-57.5001 1358.92,-64.5001\"/>\n",
       "</g>\n",
       "<!-- final_linear -->\n",
       "<g id=\"node17\" class=\"node\"><title>final_linear</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: Parameter containing:\\ntensor([ 0.0188, &#45;0.0177, &#45;0.0432, &#160;0.0172, &#160;0.0433, &#45;0.0322, &#45;0.0424, &#160;0.0387,\\n &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#45;0.0120, &#45;0.0325], requires_grad=True)}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M1501,-79C1501,-79 1471,-79 1471,-79 1465,-79 1459,-73 1459,-67 1459,-67 1459,-55 1459,-55 1459,-49 1465,-43 1471,-43 1471,-43 1501,-43 1501,-43 1507,-43 1513,-49 1513,-55 1513,-55 1513,-67 1513,-67 1513,-73 1507,-79 1501,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"1486\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_flatten&#45;&gt;final_linear -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>final_flatten&#45;&gt;final_linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1423.4,-61C1431.39,-61 1440.31,-61 1448.82,-61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1448.92,-64.5001 1458.92,-61 1448.92,-57.5001 1448.92,-64.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node18\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-65C42,-65 12,-65 12,-65 6,-65 0,-59 0,-53 0,-53 0,-41 0,-41 0,-35 6,-29 12,-29 12,-29 42,-29 42,-29 48,-29 54,-35 54,-41 54,-41 54,-53 54,-53 54,-59 48,-65 42,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-47C62.3932,-47 71.3106,-47 79.8241,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-50.5001 89.919,-47 79.919,-43.5001 79.919,-50.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7fcd23c9e898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.1000       2.0508       1.7308       0.3705       0.1861       1.3731       0.4903       2.2369\n",
      "           2       0.2000       1.5441       1.3386       0.5154       0.1478       1.1711       0.5787       3.9288\n",
      "           3       0.3000       1.7685       1.1407       0.5904       0.1450       1.0622       0.6204       5.8424\n",
      "           4       0.4000       1.6975       1.0908       0.6144       0.1506       1.0718       0.6245       7.6905\n",
      "           5       0.3750       1.7146       1.0078       0.6450       0.1492       0.9757       0.6593       9.5543\n",
      "           6       0.3500       1.6656       0.9080       0.6802       0.1534       0.8884       0.6895      11.3733\n",
      "           7       0.3250       1.7620       0.8637       0.6964       0.1456       0.9482       0.6865      13.2808\n",
      "           8       0.3000       1.5897       0.7945       0.7212       0.1450       0.7550       0.7410      15.0156\n",
      "           9       0.2750       1.6692       0.7741       0.7283       0.1356       0.7095       0.7495      16.8205\n",
      "          10       0.2500       1.5464       0.7320       0.7435       0.1475       0.7118       0.7507      18.5144\n",
      "          11       0.2250       1.7490       0.7023       0.7546       0.1470       0.6569       0.7727      20.4104\n",
      "          12       0.2000       1.7296       0.6709       0.7640       0.1654       0.6825       0.7686      22.3054\n",
      "          13       0.1750       1.8288       0.6368       0.7780       0.1482       0.6276       0.7804      24.2824\n",
      "          14       0.1500       1.7047       0.6083       0.7877       0.1469       0.6178       0.7857      26.1340\n",
      "          15       0.1250       1.7850       0.5843       0.7966       0.1479       0.5544       0.8084      28.0669\n",
      "          16       0.1000       1.8049       0.5470       0.8105       0.1589       0.5189       0.8220      30.0307\n",
      "          17       0.0750       1.8134       0.5176       0.8220       0.1542       0.5377       0.8134      31.9983\n",
      "          18       0.0500       1.7970       0.4925       0.8298       0.1479       0.4628       0.8391      33.9432\n",
      "          19       0.0250       1.6910       0.4542       0.8437       0.2026       0.4321       0.8499      35.8369\n",
      "          20       0.0000       1.6946       0.4268       0.8526       0.1461       0.4180       0.8564      37.6775\n"
     ]
    }
   ],
   "source": [
    "def shortcut_block(c_in, c_out, stride, **kw):\n",
    "    projection = (stride != 1) or (c_in != c_out)\n",
    "    if projection:\n",
    "        return {\n",
    "            'conv': nn.Conv2d(c_in, c_out, kernel_size=3, stride=stride, padding=1, bias=False), \n",
    "            'bn': batch_norm(c_out, **kw),\n",
    "            'relu': nn.ReLU(True),\n",
    "        }\n",
    "    else:\n",
    "        return {'id': Identity()}\n",
    "\n",
    "lr_schedule = PiecewiseLinear([0, 4, 20], [0, 0.4, 0])\n",
    "batch_size = 512\n",
    "\n",
    "n = DAWN_net(block=shortcut_block, prep_bn_relu=True)\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(8,8)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 4: Architecture](https://www.myrtle.ai/2018/10/26/how_to_train_your_resnet_4/)  - backbone, maxpool downsampling (43s; test acc 89.7%)\n",
    "\n",
    "We can further improve the downsampling stages by applying 3x3 convolutions of stride one followed by a pooling layer instead of using strided convolutions. \n",
    "\n",
    "We choose max pooling with a 2x2 window size leading to a final test accuracy of 89.7% after 43s. Using average pooling gives a similar result but takes slightly longer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"91pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 91.44\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.597676 0.597676) rotate(0) translate(4 149)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-149 1803,-149 1803,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-21C94,-21 320,-21 320,-21 326,-21 332,-27 332,-33 332,-33 332,-84 332,-84 332,-90 326,-96 320,-96 320,-96 94,-96 94,-96 88,-96 82,-90 82,-84 82,-84 82,-33 82,-33 82,-27 88,-21 94,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M356,-13C356,-13 688,-13 688,-13 694,-13 700,-19 700,-25 700,-25 700,-115 700,-115 700,-121 694,-127 688,-127 688,-127 356,-127 356,-127 350,-127 344,-121 344,-115 344,-115 344,-25 344,-25 344,-19 350,-13 356,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer2_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-21C364,-21 680,-21 680,-21 686,-21 692,-27 692,-33 692,-33 692,-84 692,-84 692,-90 686,-96 680,-96 680,-96 364,-96 364,-96 358,-96 352,-90 352,-84 352,-84 352,-33 352,-33 352,-27 358,-21 364,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M720,-13C720,-13 1052,-13 1052,-13 1058,-13 1064,-19 1064,-25 1064,-25 1064,-115 1064,-115 1064,-121 1058,-127 1052,-127 1052,-127 720,-127 720,-127 714,-127 708,-121 708,-115 708,-115 708,-25 708,-25 708,-19 714,-13 720,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"886\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer3_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M728,-21C728,-21 1044,-21 1044,-21 1050,-21 1056,-27 1056,-33 1056,-33 1056,-84 1056,-84 1056,-90 1050,-96 1044,-96 1044,-96 728,-96 728,-96 722,-96 716,-90 716,-84 716,-84 716,-33 716,-33 716,-27 722,-21 728,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"886\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer4</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1084,-13C1084,-13 1416,-13 1416,-13 1422,-13 1428,-19 1428,-25 1428,-25 1428,-115 1428,-115 1428,-121 1422,-127 1416,-127 1416,-127 1084,-127 1084,-127 1078,-127 1072,-121 1072,-115 1072,-115 1072,-25 1072,-25 1072,-19 1078,-13 1084,-13\"/>\n",
       "<text text-anchor=\"middle\" x=\"1250\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer4</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer4_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1092,-21C1092,-21 1408,-21 1408,-21 1414,-21 1420,-27 1420,-33 1420,-33 1420,-84 1420,-84 1420,-90 1414,-96 1408,-96 1408,-96 1092,-96 1092,-96 1086,-96 1080,-90 1080,-84 1080,-84 1080,-33 1080,-33 1080,-27 1086,-21 1092,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"1250\" y=\"-80.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_final</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1452,-8C1452,-8 1779,-8 1779,-8 1785,-8 1791,-14 1791,-20 1791,-20 1791,-125 1791,-125 1791,-131 1785,-137 1779,-137 1779,-137 1452,-137 1452,-137 1446,-137 1440,-131 1440,-125 1440,-125 1440,-20 1440,-20 1440,-14 1446,-8 1452,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"1615.5\" y=\"-121.8\" font-family=\"Times,serif\" font-size=\"14.00\">final</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-65C132,-65 102,-65 102,-65 96,-65 90,-59 90,-53 90,-53 90,-41 90,-41 90,-35 96,-29 102,-29 102,-29 132,-29 132,-29 138,-29 144,-35 144,-41 144,-41 144,-53 144,-53 144,-59 138,-65 132,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn -->\n",
       "<g id=\"node2\" class=\"node\"><title>prep_bn</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-65C222,-65 192,-65 192,-65 186,-65 180,-59 180,-53 180,-53 180,-41 180,-41 180,-35 186,-29 192,-29 192,-29 222,-29 222,-29 228,-29 234,-35 234,-41 234,-41 234,-53 234,-53 234,-59 228,-65 222,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;prep_bn -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;prep_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-47C152.393,-47 161.311,-47 169.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-50.5001 179.919,-47 169.919,-43.5001 169.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- prep_relu -->\n",
       "<g id=\"node3\" class=\"node\"><title>prep_relu</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-65C312,-65 282,-65 282,-65 276,-65 270,-59 270,-53 270,-53 270,-41 270,-41 270,-35 276,-29 282,-29 282,-29 312,-29 312,-29 318,-29 324,-35 324,-41 324,-41 324,-53 324,-53 324,-59 318,-65 312,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn&#45;&gt;prep_relu -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>prep_bn&#45;&gt;prep_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-47C242.393,-47 251.311,-47 259.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-50.5001 269.919,-47 259.919,-43.5001 259.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer2_block0_conv</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-65C402,-65 372,-65 372,-65 366,-65 360,-59 360,-53 360,-53 360,-41 360,-41 360,-35 366,-29 372,-29 372,-29 402,-29 402,-29 408,-29 414,-35 414,-41 414,-41 414,-53 414,-53 414,-59 408,-65 402,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_relu&#45;&gt;layer2_block0_conv -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>prep_relu&#45;&gt;layer2_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-47C332.393,-47 341.311,-47 349.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.919,-50.5001 359.919,-47 349.919,-43.5001 349.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_bn -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer2_block0_bn</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-65C492,-65 462,-65 462,-65 456,-65 450,-59 450,-53 450,-53 450,-41 450,-41 450,-35 456,-29 462,-29 462,-29 492,-29 492,-29 498,-29 504,-35 504,-41 504,-41 504,-53 504,-53 504,-59 498,-65 492,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_conv&#45;&gt;layer2_block0_bn -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer2_block0_conv&#45;&gt;layer2_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-47C422.393,-47 431.311,-47 439.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-50.5001 449.919,-47 439.919,-43.5001 439.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_relu -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer2_block0_relu</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-65C582,-65 552,-65 552,-65 546,-65 540,-59 540,-53 540,-53 540,-41 540,-41 540,-35 546,-29 552,-29 552,-29 582,-29 582,-29 588,-29 594,-35 594,-41 594,-41 594,-53 594,-53 594,-59 588,-65 582,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_bn&#45;&gt;layer2_block0_relu -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer2_block0_bn&#45;&gt;layer2_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-47C512.393,-47 521.311,-47 529.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-50.5001 539.919,-47 529.919,-43.5001 529.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_pool -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer2_block0_pool</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M672,-65C672,-65 642,-65 642,-65 636,-65 630,-59 630,-53 630,-53 630,-41 630,-41 630,-35 636,-29 642,-29 642,-29 672,-29 672,-29 678,-29 684,-35 684,-41 684,-41 684,-53 684,-53 684,-59 678,-65 672,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu&#45;&gt;layer2_block0_pool -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer2_block0_relu&#45;&gt;layer2_block0_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-47C602.393,-47 611.311,-47 619.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-50.5001 629.919,-47 619.919,-43.5001 619.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer3_block0_conv</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M766,-65C766,-65 736,-65 736,-65 730,-65 724,-59 724,-53 724,-53 724,-41 724,-41 724,-35 730,-29 736,-29 736,-29 766,-29 766,-29 772,-29 778,-35 778,-41 778,-41 778,-53 778,-53 778,-59 772,-65 766,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"751\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_pool&#45;&gt;layer3_block0_conv -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer2_block0_pool&#45;&gt;layer3_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.117,-47C693.343,-47 703.891,-47 713.802,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"713.817,-50.5001 723.817,-47 713.817,-43.5001 713.817,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_bn -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer3_block0_bn</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M856,-65C856,-65 826,-65 826,-65 820,-65 814,-59 814,-53 814,-53 814,-41 814,-41 814,-35 820,-29 826,-29 826,-29 856,-29 856,-29 862,-29 868,-35 868,-41 868,-41 868,-53 868,-53 868,-59 862,-65 856,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"841\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_conv&#45;&gt;layer3_block0_bn -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer3_block0_conv&#45;&gt;layer3_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M778.403,-47C786.393,-47 795.311,-47 803.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"803.919,-50.5001 813.919,-47 803.919,-43.5001 803.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_relu -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer3_block0_relu</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M946,-65C946,-65 916,-65 916,-65 910,-65 904,-59 904,-53 904,-53 904,-41 904,-41 904,-35 910,-29 916,-29 916,-29 946,-29 946,-29 952,-29 958,-35 958,-41 958,-41 958,-53 958,-53 958,-59 952,-65 946,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"931\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_bn&#45;&gt;layer3_block0_relu -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer3_block0_bn&#45;&gt;layer3_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M868.403,-47C876.393,-47 885.311,-47 893.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"893.919,-50.5001 903.919,-47 893.919,-43.5001 893.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_pool -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer3_block0_pool</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1036,-65C1036,-65 1006,-65 1006,-65 1000,-65 994,-59 994,-53 994,-53 994,-41 994,-41 994,-35 1000,-29 1006,-29 1006,-29 1036,-29 1036,-29 1042,-29 1048,-35 1048,-41 1048,-41 1048,-53 1048,-53 1048,-59 1042,-65 1036,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1021\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu&#45;&gt;layer3_block0_pool -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer3_block0_relu&#45;&gt;layer3_block0_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M958.403,-47C966.393,-47 975.311,-47 983.824,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"983.919,-50.5001 993.919,-47 983.919,-43.5001 983.919,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer4_block0_conv</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1130,-65C1130,-65 1100,-65 1100,-65 1094,-65 1088,-59 1088,-53 1088,-53 1088,-41 1088,-41 1088,-35 1094,-29 1100,-29 1100,-29 1130,-29 1130,-29 1136,-29 1142,-35 1142,-41 1142,-41 1142,-53 1142,-53 1142,-59 1136,-65 1130,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1115\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_pool&#45;&gt;layer4_block0_conv -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer3_block0_pool&#45;&gt;layer4_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1048.12,-47C1057.34,-47 1067.89,-47 1077.8,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1077.82,-50.5001 1087.82,-47 1077.82,-43.5001 1077.82,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_bn -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer4_block0_bn</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1220,-65C1220,-65 1190,-65 1190,-65 1184,-65 1178,-59 1178,-53 1178,-53 1178,-41 1178,-41 1178,-35 1184,-29 1190,-29 1190,-29 1220,-29 1220,-29 1226,-29 1232,-35 1232,-41 1232,-41 1232,-53 1232,-53 1232,-59 1226,-65 1220,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1205\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_conv&#45;&gt;layer4_block0_bn -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer4_block0_conv&#45;&gt;layer4_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1142.4,-47C1150.39,-47 1159.31,-47 1167.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1167.92,-50.5001 1177.92,-47 1167.92,-43.5001 1167.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_relu -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer4_block0_relu</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1310,-65C1310,-65 1280,-65 1280,-65 1274,-65 1268,-59 1268,-53 1268,-53 1268,-41 1268,-41 1268,-35 1274,-29 1280,-29 1280,-29 1310,-29 1310,-29 1316,-29 1322,-35 1322,-41 1322,-41 1322,-53 1322,-53 1322,-59 1316,-65 1310,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1295\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_bn&#45;&gt;layer4_block0_relu -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer4_block0_bn&#45;&gt;layer4_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1232.4,-47C1240.39,-47 1249.31,-47 1257.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1257.92,-50.5001 1267.92,-47 1257.92,-43.5001 1257.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_pool -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer4_block0_pool</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1400,-65C1400,-65 1370,-65 1370,-65 1364,-65 1358,-59 1358,-53 1358,-53 1358,-41 1358,-41 1358,-35 1364,-29 1370,-29 1370,-29 1400,-29 1400,-29 1406,-29 1412,-35 1412,-41 1412,-41 1412,-53 1412,-53 1412,-59 1406,-65 1400,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"1385\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu&#45;&gt;layer4_block0_pool -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer4_block0_relu&#45;&gt;layer4_block0_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1322.4,-47C1330.39,-47 1339.31,-47 1347.82,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1347.92,-50.5001 1357.92,-47 1347.92,-43.5001 1347.92,-50.5001\"/>\n",
       "</g>\n",
       "<!-- final_maxpool -->\n",
       "<g id=\"node16\" class=\"node\"><title>final_maxpool</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1501,-106C1501,-106 1460,-106 1460,-106 1454,-106 1448,-100 1448,-94 1448,-94 1448,-82 1448,-82 1448,-76 1454,-70 1460,-70 1460,-70 1501,-70 1501,-70 1507,-70 1513,-76 1513,-82 1513,-82 1513,-94 1513,-94 1513,-100 1507,-106 1501,-106\"/>\n",
       "<text text-anchor=\"middle\" x=\"1480.5\" y=\"-84.3\" font-family=\"Times,serif\" font-size=\"14.00\">maxpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_pool&#45;&gt;final_maxpool -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer4_block0_pool&#45;&gt;final_maxpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1412.04,-58.4208C1420.31,-62.0434 1429.65,-66.1401 1438.69,-70.1047\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1437.38,-73.3525 1447.94,-74.163 1440.19,-66.9417 1437.38,-73.3525\"/>\n",
       "</g>\n",
       "<!-- final_avgpool -->\n",
       "<g id=\"node17\" class=\"node\"><title>final_avgpool</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"AvgPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;ceil_mode&#39;: False, &#39;count_include_pad&#39;: True}\">\n",
       "<path fill=\"#80b1d3\" stroke=\"black\" d=\"M1498.5,-52C1498.5,-52 1462.5,-52 1462.5,-52 1456.5,-52 1450.5,-46 1450.5,-40 1450.5,-40 1450.5,-28 1450.5,-28 1450.5,-22 1456.5,-16 1462.5,-16 1462.5,-16 1498.5,-16 1498.5,-16 1504.5,-16 1510.5,-22 1510.5,-28 1510.5,-28 1510.5,-40 1510.5,-40 1510.5,-46 1504.5,-52 1498.5,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1480.5\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">avgpool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_pool&#45;&gt;final_avgpool -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>layer4_block0_pool&#45;&gt;final_avgpool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1412.04,-43.3788C1420.77,-42.165 1430.71,-40.7833 1440.23,-39.4606\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1440.88,-42.9031 1450.3,-38.0593 1439.92,-35.9698 1440.88,-42.9031\"/>\n",
       "</g>\n",
       "<!-- final_concat -->\n",
       "<g id=\"node18\" class=\"node\"><title>final_concat</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Concat {}\">\n",
       "<path fill=\"#fdb462\" stroke=\"black\" d=\"M1591,-79C1591,-79 1561,-79 1561,-79 1555,-79 1549,-73 1549,-67 1549,-67 1549,-55 1549,-55 1549,-49 1555,-43 1561,-43 1561,-43 1591,-43 1591,-43 1597,-43 1603,-49 1603,-55 1603,-55 1603,-67 1603,-67 1603,-73 1597,-79 1591,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"1576\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">concat</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_maxpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>final_maxpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1513.15,-78.859C1521.53,-76.4399 1530.63,-73.8116 1539.21,-71.3341\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1540.33,-74.6535 1548.97,-68.5165 1538.39,-67.9283 1540.33,-74.6535\"/>\n",
       "</g>\n",
       "<!-- final_avgpool&#45;&gt;final_concat -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>final_avgpool&#45;&gt;final_concat</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1510.57,-42.395C1519.55,-44.9869 1529.54,-47.8734 1538.93,-50.5845\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1538.22,-54.0234 1548.8,-53.4352 1540.17,-47.2982 1538.22,-54.0234\"/>\n",
       "</g>\n",
       "<!-- final_flatten -->\n",
       "<g id=\"node19\" class=\"node\"><title>final_flatten</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M1681,-79C1681,-79 1651,-79 1651,-79 1645,-79 1639,-73 1639,-67 1639,-67 1639,-55 1639,-55 1639,-49 1645,-43 1651,-43 1651,-43 1681,-43 1681,-43 1687,-43 1693,-49 1693,-55 1693,-55 1693,-67 1693,-67 1693,-73 1687,-79 1681,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"1666\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_concat&#45;&gt;final_flatten -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>final_concat&#45;&gt;final_flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1603.4,-61C1611.39,-61 1620.31,-61 1628.82,-61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1628.92,-64.5001 1638.92,-61 1628.92,-57.5001 1628.92,-64.5001\"/>\n",
       "</g>\n",
       "<!-- final_linear -->\n",
       "<g id=\"node20\" class=\"node\"><title>final_linear</title>\n",
       "<g id=\"a_node20\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: Parameter containing:\\ntensor([&#45;0.0320, &#45;0.0227, &#45;0.0181, &#160;0.0112, &#45;0.0310, &#45;0.0283, &#45;0.0323, &#160;0.0097,\\n &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#45;0.0304, &#45;0.0037], requires_grad=True)}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M1771,-79C1771,-79 1741,-79 1741,-79 1735,-79 1729,-73 1729,-67 1729,-67 1729,-55 1729,-55 1729,-49 1735,-43 1741,-43 1741,-43 1771,-43 1771,-43 1777,-43 1783,-49 1783,-55 1783,-55 1783,-67 1783,-67 1783,-73 1777,-79 1771,-79\"/>\n",
       "<text text-anchor=\"middle\" x=\"1756\" y=\"-57.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_flatten&#45;&gt;final_linear -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>final_flatten&#45;&gt;final_linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1693.4,-61C1701.39,-61 1710.31,-61 1718.82,-61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1718.92,-64.5001 1728.92,-61 1718.92,-57.5001 1718.92,-64.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node21\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-65C42,-65 12,-65 12,-65 6,-65 0,-59 0,-53 0,-53 0,-41 0,-41 0,-35 6,-29 12,-29 12,-29 42,-29 42,-29 48,-29 54,-35 54,-41 54,-41 54,-53 54,-53 54,-59 48,-65 42,-65\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-43.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-47C62.3932,-47 71.3106,-47 79.8241,-47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-50.5001 89.919,-47 79.919,-43.5001 79.919,-50.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7fcd23d18e48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.1000       3.0536       1.6871       0.3949       0.2558       1.3948       0.5405       3.3093\n",
      "           2       0.2000       1.8671       1.1606       0.5894       0.1508       0.9039       0.6839       5.3273\n",
      "           3       0.3000       1.9303       0.9527       0.6686       0.1573       1.0704       0.6257       7.4150\n",
      "           4       0.4000       1.8663       0.8934       0.6901       0.1485       0.8327       0.7113       9.4297\n",
      "           5       0.3750       1.8682       0.8073       0.7195       0.1412       0.7715       0.7316      11.4391\n",
      "           6       0.3500       1.8614       0.7249       0.7520       0.1484       0.6337       0.7838      13.4489\n",
      "           7       0.3250       1.9188       0.6648       0.7700       0.1519       0.7069       0.7600      15.5196\n",
      "           8       0.3000       1.8644       0.6291       0.7826       0.1530       0.6076       0.7903      17.5370\n",
      "           9       0.2750       1.8652       0.5798       0.7996       0.1499       0.6217       0.7860      19.5520\n",
      "          10       0.2500       1.8641       0.5386       0.8144       0.1500       0.5035       0.8329      21.5662\n",
      "          11       0.2250       1.9066       0.5156       0.8223       0.1424       0.7209       0.7544      23.6152\n",
      "          12       0.2000       1.8662       0.4905       0.8302       0.1561       0.6311       0.7924      25.6374\n",
      "          13       0.1750       1.8650       0.4566       0.8434       0.1529       0.4874       0.8386      27.6554\n",
      "          14       0.1500       1.8657       0.4312       0.8514       0.1516       0.4225       0.8643      29.6727\n",
      "          15       0.1250       1.9064       0.3978       0.8625       0.1514       0.4320       0.8532      31.7306\n",
      "          16       0.1000       1.8770       0.3756       0.8704       0.1599       0.4019       0.8656      33.7675\n",
      "          17       0.0750       1.8797       0.3426       0.8825       0.1469       0.3933       0.8708      35.7942\n",
      "          18       0.0500       1.8622       0.3104       0.8940       0.1525       0.3574       0.8788      37.8089\n",
      "          19       0.0250       1.9173       0.2823       0.9027       0.1508       0.3317       0.8905      39.8770\n",
      "          20       0.0000       1.8654       0.2529       0.9131       0.1529       0.3054       0.8971      41.8954\n"
     ]
    }
   ],
   "source": [
    "def shortcut_block(c_in, c_out, stride, **kw):\n",
    "    projection = (stride != 1) or (c_in != c_out)\n",
    "    if projection:\n",
    "        return {\n",
    "            'conv': nn.Conv2d(c_in, c_out, kernel_size=3, stride=1, padding=1, bias=False), \n",
    "            'bn': batch_norm(c_out, **kw),\n",
    "            'relu': nn.ReLU(True),\n",
    "            'pool': nn.MaxPool2d(2),\n",
    "        }\n",
    "    else:\n",
    "        return {'id': Identity()}\n",
    "\n",
    "lr_schedule = PiecewiseLinear([0, 4, 20], [0, 0.4, 0])\n",
    "batch_size = 512\n",
    "\n",
    "n = DAWN_net(block=shortcut_block, prep_bn_relu=True)\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(8,8)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 4: Architecture](https://www.myrtle.ai/2018/10/26/how_to_train_your_resnet_4/)  - backbone, 2x output dim, global maxpool (47s; test acc 90.7%)\n",
    "\n",
    "The final pooling layer before the classifier is a concatenation of global average pooling and max pooling layers, inherited from the original network. \n",
    "\n",
    "We replace this with a more standard global max pooling layer and double the output dimension of the final convolution to compensate for the reduction in input dimension to the classifier, leading to a final test accuracy of 90.7% in 47s. Note that average pooling at this stage underperforms max pooling significantly.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"87pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 87.36\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.63306 0.63306) rotate(0) translate(4 134)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-134 1702,-134 1702,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-16C94,-16 320,-16 320,-16 326,-16 332,-22 332,-28 332,-28 332,-79 332,-79 332,-85 326,-91 320,-91 320,-91 94,-91 94,-91 88,-91 82,-85 82,-79 82,-79 82,-28 82,-28 82,-22 88,-16 94,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M356,-8C356,-8 688,-8 688,-8 694,-8 700,-14 700,-20 700,-20 700,-110 700,-110 700,-116 694,-122 688,-122 688,-122 356,-122 356,-122 350,-122 344,-116 344,-110 344,-110 344,-20 344,-20 344,-14 350,-8 356,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-106.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer2_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-16C364,-16 680,-16 680,-16 686,-16 692,-22 692,-28 692,-28 692,-79 692,-79 692,-85 686,-91 680,-91 680,-91 364,-91 364,-91 358,-91 352,-85 352,-79 352,-79 352,-28 352,-28 352,-22 358,-16 364,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M720,-8C720,-8 1052,-8 1052,-8 1058,-8 1064,-14 1064,-20 1064,-20 1064,-110 1064,-110 1064,-116 1058,-122 1052,-122 1052,-122 720,-122 720,-122 714,-122 708,-116 708,-110 708,-110 708,-20 708,-20 708,-14 714,-8 720,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"886\" y=\"-106.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer3_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M728,-16C728,-16 1044,-16 1044,-16 1050,-16 1056,-22 1056,-28 1056,-28 1056,-79 1056,-79 1056,-85 1050,-91 1044,-91 1044,-91 728,-91 728,-91 722,-91 716,-85 716,-79 716,-79 716,-28 716,-28 716,-22 722,-16 728,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"886\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer4</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1084,-8C1084,-8 1416,-8 1416,-8 1422,-8 1428,-14 1428,-20 1428,-20 1428,-110 1428,-110 1428,-116 1422,-122 1416,-122 1416,-122 1084,-122 1084,-122 1078,-122 1072,-116 1072,-110 1072,-110 1072,-20 1072,-20 1072,-14 1078,-8 1084,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"1250\" y=\"-106.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer4</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer4_block0</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1092,-16C1092,-16 1408,-16 1408,-16 1414,-16 1420,-22 1420,-28 1420,-28 1420,-79 1420,-79 1420,-85 1414,-91 1408,-91 1408,-91 1092,-91 1092,-91 1086,-91 1080,-85 1080,-79 1080,-79 1080,-28 1080,-28 1080,-22 1086,-16 1092,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"1250\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">block0</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_final</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1452,-16C1452,-16 1678,-16 1678,-16 1684,-16 1690,-22 1690,-28 1690,-28 1690,-79 1690,-79 1690,-85 1684,-91 1678,-91 1678,-91 1452,-91 1452,-91 1446,-91 1440,-85 1440,-79 1440,-79 1440,-28 1440,-28 1440,-22 1446,-16 1452,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"1565\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">final</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-60C132,-60 102,-60 102,-60 96,-60 90,-54 90,-48 90,-48 90,-36 90,-36 90,-30 96,-24 102,-24 102,-24 132,-24 132,-24 138,-24 144,-30 144,-36 144,-36 144,-48 144,-48 144,-54 138,-60 132,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn -->\n",
       "<g id=\"node2\" class=\"node\"><title>prep_bn</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-60C222,-60 192,-60 192,-60 186,-60 180,-54 180,-48 180,-48 180,-36 180,-36 180,-30 186,-24 192,-24 192,-24 222,-24 222,-24 228,-24 234,-30 234,-36 234,-36 234,-48 234,-48 234,-54 228,-60 222,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;prep_bn -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;prep_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-42C152.393,-42 161.311,-42 169.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-45.5001 179.919,-42 169.919,-38.5001 169.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- prep_relu -->\n",
       "<g id=\"node3\" class=\"node\"><title>prep_relu</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-60C312,-60 282,-60 282,-60 276,-60 270,-54 270,-48 270,-48 270,-36 270,-36 270,-30 276,-24 282,-24 282,-24 312,-24 312,-24 318,-24 324,-30 324,-36 324,-36 324,-48 324,-48 324,-54 318,-60 312,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn&#45;&gt;prep_relu -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>prep_bn&#45;&gt;prep_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-42C242.393,-42 251.311,-42 259.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-45.5001 269.919,-42 259.919,-38.5001 259.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_conv -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer2_block0_conv</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-60C402,-60 372,-60 372,-60 366,-60 360,-54 360,-48 360,-48 360,-36 360,-36 360,-30 366,-24 372,-24 372,-24 402,-24 402,-24 408,-24 414,-30 414,-36 414,-36 414,-48 414,-48 414,-54 408,-60 402,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_relu&#45;&gt;layer2_block0_conv -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>prep_relu&#45;&gt;layer2_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-42C332.393,-42 341.311,-42 349.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.919,-45.5001 359.919,-42 349.919,-38.5001 349.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_bn -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer2_block0_bn</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-60C492,-60 462,-60 462,-60 456,-60 450,-54 450,-48 450,-48 450,-36 450,-36 450,-30 456,-24 462,-24 462,-24 492,-24 492,-24 498,-24 504,-30 504,-36 504,-36 504,-48 504,-48 504,-54 498,-60 492,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_conv&#45;&gt;layer2_block0_bn -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer2_block0_conv&#45;&gt;layer2_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-42C422.393,-42 431.311,-42 439.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-45.5001 449.919,-42 439.919,-38.5001 439.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_relu -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer2_block0_relu</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-60C582,-60 552,-60 552,-60 546,-60 540,-54 540,-48 540,-48 540,-36 540,-36 540,-30 546,-24 552,-24 552,-24 582,-24 582,-24 588,-24 594,-30 594,-36 594,-36 594,-48 594,-48 594,-54 588,-60 582,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_bn&#45;&gt;layer2_block0_relu -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer2_block0_bn&#45;&gt;layer2_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-42C512.393,-42 521.311,-42 529.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-45.5001 539.919,-42 529.919,-38.5001 529.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_block0_pool -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer2_block0_pool</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M672,-60C672,-60 642,-60 642,-60 636,-60 630,-54 630,-48 630,-48 630,-36 630,-36 630,-30 636,-24 642,-24 642,-24 672,-24 672,-24 678,-24 684,-30 684,-36 684,-36 684,-48 684,-48 684,-54 678,-60 672,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_relu&#45;&gt;layer2_block0_pool -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer2_block0_relu&#45;&gt;layer2_block0_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-42C602.393,-42 611.311,-42 619.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-45.5001 629.919,-42 619.919,-38.5001 619.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_conv -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer3_block0_conv</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M766,-60C766,-60 736,-60 736,-60 730,-60 724,-54 724,-48 724,-48 724,-36 724,-36 724,-30 730,-24 736,-24 736,-24 766,-24 766,-24 772,-24 778,-30 778,-36 778,-36 778,-48 778,-48 778,-54 772,-60 766,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"751\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_block0_pool&#45;&gt;layer3_block0_conv -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer2_block0_pool&#45;&gt;layer3_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.117,-42C693.343,-42 703.891,-42 713.802,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"713.817,-45.5001 723.817,-42 713.817,-38.5001 713.817,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_bn -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer3_block0_bn</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M856,-60C856,-60 826,-60 826,-60 820,-60 814,-54 814,-48 814,-48 814,-36 814,-36 814,-30 820,-24 826,-24 826,-24 856,-24 856,-24 862,-24 868,-30 868,-36 868,-36 868,-48 868,-48 868,-54 862,-60 856,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"841\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_conv&#45;&gt;layer3_block0_bn -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer3_block0_conv&#45;&gt;layer3_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M778.403,-42C786.393,-42 795.311,-42 803.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"803.919,-45.5001 813.919,-42 803.919,-38.5001 803.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_relu -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer3_block0_relu</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M946,-60C946,-60 916,-60 916,-60 910,-60 904,-54 904,-48 904,-48 904,-36 904,-36 904,-30 910,-24 916,-24 916,-24 946,-24 946,-24 952,-24 958,-30 958,-36 958,-36 958,-48 958,-48 958,-54 952,-60 946,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"931\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_bn&#45;&gt;layer3_block0_relu -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer3_block0_bn&#45;&gt;layer3_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M868.403,-42C876.393,-42 885.311,-42 893.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"893.919,-45.5001 903.919,-42 893.919,-38.5001 893.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_block0_pool -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer3_block0_pool</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1036,-60C1036,-60 1006,-60 1006,-60 1000,-60 994,-54 994,-48 994,-48 994,-36 994,-36 994,-30 1000,-24 1006,-24 1006,-24 1036,-24 1036,-24 1042,-24 1048,-30 1048,-36 1048,-36 1048,-48 1048,-48 1048,-54 1042,-60 1036,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1021\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_relu&#45;&gt;layer3_block0_pool -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer3_block0_relu&#45;&gt;layer3_block0_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M958.403,-42C966.393,-42 975.311,-42 983.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"983.919,-45.5001 993.919,-42 983.919,-38.5001 983.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_conv -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer4_block0_conv</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 512, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1130,-60C1130,-60 1100,-60 1100,-60 1094,-60 1088,-54 1088,-48 1088,-48 1088,-36 1088,-36 1088,-30 1094,-24 1100,-24 1100,-24 1130,-24 1130,-24 1136,-24 1142,-30 1142,-36 1142,-36 1142,-48 1142,-48 1142,-54 1136,-60 1130,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1115\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_block0_pool&#45;&gt;layer4_block0_conv -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer3_block0_pool&#45;&gt;layer4_block0_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1048.12,-42C1057.34,-42 1067.89,-42 1077.8,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1077.82,-45.5001 1087.82,-42 1077.82,-38.5001 1077.82,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_bn -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer4_block0_bn</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 512, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1220,-60C1220,-60 1190,-60 1190,-60 1184,-60 1178,-54 1178,-48 1178,-48 1178,-36 1178,-36 1178,-30 1184,-24 1190,-24 1190,-24 1220,-24 1220,-24 1226,-24 1232,-30 1232,-36 1232,-36 1232,-48 1232,-48 1232,-54 1226,-60 1220,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1205\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_conv&#45;&gt;layer4_block0_bn -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer4_block0_conv&#45;&gt;layer4_block0_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1142.4,-42C1150.39,-42 1159.31,-42 1167.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1167.92,-45.5001 1177.92,-42 1167.92,-38.5001 1167.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_relu -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer4_block0_relu</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1310,-60C1310,-60 1280,-60 1280,-60 1274,-60 1268,-54 1268,-48 1268,-48 1268,-36 1268,-36 1268,-30 1274,-24 1280,-24 1280,-24 1310,-24 1310,-24 1316,-24 1322,-30 1322,-36 1322,-36 1322,-48 1322,-48 1322,-54 1316,-60 1310,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1295\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_bn&#45;&gt;layer4_block0_relu -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer4_block0_bn&#45;&gt;layer4_block0_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1232.4,-42C1240.39,-42 1249.31,-42 1257.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1257.92,-45.5001 1267.92,-42 1257.92,-38.5001 1257.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer4_block0_pool -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer4_block0_pool</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1400,-60C1400,-60 1370,-60 1370,-60 1364,-60 1358,-54 1358,-48 1358,-48 1358,-36 1358,-36 1358,-30 1364,-24 1370,-24 1370,-24 1400,-24 1400,-24 1406,-24 1412,-30 1412,-36 1412,-36 1412,-48 1412,-48 1412,-54 1406,-60 1400,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1385\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_relu&#45;&gt;layer4_block0_pool -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer4_block0_relu&#45;&gt;layer4_block0_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1322.4,-42C1330.39,-42 1339.31,-42 1347.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1347.92,-45.5001 1357.92,-42 1347.92,-38.5001 1347.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- final_pool -->\n",
       "<g id=\"node16\" class=\"node\"><title>final_pool</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1490,-60C1490,-60 1460,-60 1460,-60 1454,-60 1448,-54 1448,-48 1448,-48 1448,-36 1448,-36 1448,-30 1454,-24 1460,-24 1460,-24 1490,-24 1490,-24 1496,-24 1502,-30 1502,-36 1502,-36 1502,-48 1502,-48 1502,-54 1496,-60 1490,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1475\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer4_block0_pool&#45;&gt;final_pool -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer4_block0_pool&#45;&gt;final_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1412.4,-42C1420.39,-42 1429.31,-42 1437.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1437.92,-45.5001 1447.92,-42 1437.92,-38.5001 1437.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- final_flatten -->\n",
       "<g id=\"node17\" class=\"node\"><title>final_flatten</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M1580,-60C1580,-60 1550,-60 1550,-60 1544,-60 1538,-54 1538,-48 1538,-48 1538,-36 1538,-36 1538,-30 1544,-24 1550,-24 1550,-24 1580,-24 1580,-24 1586,-24 1592,-30 1592,-36 1592,-36 1592,-48 1592,-48 1592,-54 1586,-60 1580,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1565\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_pool&#45;&gt;final_flatten -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>final_pool&#45;&gt;final_flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1502.4,-42C1510.39,-42 1519.31,-42 1527.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1527.92,-45.5001 1537.92,-42 1527.92,-38.5001 1527.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- final_linear -->\n",
       "<g id=\"node18\" class=\"node\"><title>final_linear</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: Parameter containing:\\ntensor([ 0.0132, &#160;0.0113, &#45;0.0333, &#160;0.0218, &#160;0.0358, &#160;0.0272, &#45;0.0194, &#45;0.0332,\\n &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#45;0.0221, &#45;0.0331], requires_grad=True)}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M1670,-60C1670,-60 1640,-60 1640,-60 1634,-60 1628,-54 1628,-48 1628,-48 1628,-36 1628,-36 1628,-30 1634,-24 1640,-24 1640,-24 1670,-24 1670,-24 1676,-24 1682,-30 1682,-36 1682,-36 1682,-48 1682,-48 1682,-54 1676,-60 1670,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1655\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- final_flatten&#45;&gt;final_linear -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>final_flatten&#45;&gt;final_linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1592.4,-42C1600.39,-42 1609.31,-42 1617.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1617.92,-45.5001 1627.92,-42 1617.92,-38.5001 1617.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node19\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-60C42,-60 12,-60 12,-60 6,-60 0,-54 0,-48 0,-48 0,-36 0,-36 0,-30 6,-24 12,-24 12,-24 42,-24 42,-24 48,-24 54,-30 54,-36 54,-36 54,-48 54,-48 54,-54 48,-60 42,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-42C62.3932,-42 71.3106,-42 79.8241,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-45.5001 89.919,-42 79.919,-38.5001 79.919,-45.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7fcd232ba550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.1000       2.4832       1.8201       0.3688       0.2066       1.3938       0.5079       2.6898\n",
      "           2       0.2000       2.0672       1.2519       0.5615       0.2158       1.0516       0.6293       4.9728\n",
      "           3       0.3000       2.0645       0.9693       0.6614       0.1586       0.7822       0.7311       7.1959\n",
      "           4       0.4000       2.0617       0.9380       0.6799       0.1542       1.3522       0.5797       9.4118\n",
      "           5       0.3750       2.0584       0.8116       0.7226       0.1445       0.7060       0.7579      11.6147\n",
      "           6       0.3500       2.0564       0.7125       0.7547       0.2066       0.6966       0.7664      13.8777\n",
      "           7       0.3250       2.0605       0.6593       0.7730       0.1547       0.6893       0.7631      16.0929\n",
      "           8       0.3000       2.0591       0.6182       0.7878       0.1519       0.5970       0.7911      18.3038\n",
      "           9       0.2750       2.0575       0.5669       0.8033       0.1537       0.7681       0.7333      20.5150\n",
      "          10       0.2500       2.0601       0.5289       0.8183       0.2198       0.6401       0.7792      22.7949\n",
      "          11       0.2250       2.0588       0.4940       0.8308       0.1544       0.4977       0.8302      25.0081\n",
      "          12       0.2000       2.0581       0.4644       0.8393       0.1515       0.4657       0.8438      27.2177\n",
      "          13       0.1750       2.0587       0.4274       0.8541       0.1518       0.4770       0.8378      29.4282\n",
      "          14       0.1500       2.0597       0.3983       0.8631       0.2062       0.4417       0.8505      31.6941\n",
      "          15       0.1250       2.0591       0.3726       0.8708       0.1552       0.3965       0.8693      33.9085\n",
      "          16       0.1000       2.0593       0.3408       0.8820       0.1532       0.3824       0.8721      36.1209\n",
      "          17       0.0750       2.0585       0.3090       0.8953       0.1528       0.3850       0.8685      38.3322\n",
      "          18       0.0500       2.0560       0.2768       0.9061       0.1438       0.3275       0.8902      40.5320\n",
      "          19       0.0250       2.1043       0.2438       0.9176       0.1542       0.2931       0.9009      42.7905\n",
      "          20       0.0000       2.0587       0.2228       0.9245       0.1516       0.2805       0.9060      45.0008\n"
     ]
    }
   ],
   "source": [
    "def shortcut_block(c_in, c_out, stride, **kw):\n",
    "    projection = (stride != 1) or (c_in != c_out)\n",
    "    if projection:\n",
    "        return {\n",
    "            'conv': nn.Conv2d(c_in, c_out, kernel_size=3, stride=1, padding=1, bias=False), \n",
    "            'bn': batch_norm(c_out, **kw),\n",
    "            'relu': nn.ReLU(True),\n",
    "            'pool': nn.MaxPool2d(2),\n",
    "        }\n",
    "    else:\n",
    "        return {'id': Identity()}\n",
    "\n",
    "lr_schedule = PiecewiseLinear([0, 4, 20], [0, 0.4, 0])\n",
    "batch_size = 512\n",
    "\n",
    "n = DAWN_net(c=[64,128,256,512], block=shortcut_block, prep_bn_relu=True, concat_pool=False)\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(8,8)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 4: Architecture](https://www.myrtle.ai/2018/10/26/how_to_train_your_resnet_4/)  - backbone, bn scale init=1, classifier weight=0.125 (47s; test acc 91.1%)\n",
    "\n",
    "By default in PyTorch (0.4), initial batch norm scales are chosen uniformly at random from the interval [0,1]. Channels which are initialised near zero could be wasted so we replace this with a constant initialisation at 1. \n",
    "This leads to a larger signal through the network and to compensate we introduce an overall constant multiplicative rescaling of the final classifier. A rough manual optimisation of this extra hyperparameter suggest that 0.125 is a reasonable value. \n",
    "(The low value makes predictions less certain and appears to ease optimisation.) \n",
    "\n",
    "With these changes in place, 20 epoch training reaches a test accuracy of 91.1% in 47s. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"60pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 59.97\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.605721 0.605721) rotate(0) translate(4 95)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-95 1779,-95 1779,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-8C94,-8 320,-8 320,-8 326,-8 332,-14 332,-20 332,-20 332,-71 332,-71 332,-77 326,-83 320,-83 320,-83 94,-83 94,-83 88,-83 82,-77 82,-71 82,-71 82,-20 82,-20 82,-14 88,-8 94,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-8C364,-8 680,-8 680,-8 686,-8 692,-14 692,-20 692,-20 692,-71 692,-71 692,-77 686,-83 680,-83 680,-83 364,-83 364,-83 358,-83 352,-77 352,-71 352,-71 352,-20 352,-20 352,-14 358,-8 364,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M724,-8C724,-8 1040,-8 1040,-8 1046,-8 1052,-14 1052,-20 1052,-20 1052,-71 1052,-71 1052,-77 1046,-83 1040,-83 1040,-83 724,-83 724,-83 718,-83 712,-77 712,-71 712,-71 712,-20 712,-20 712,-14 718,-8 724,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"882\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1084,-8C1084,-8 1400,-8 1400,-8 1406,-8 1412,-14 1412,-20 1412,-20 1412,-71 1412,-71 1412,-77 1406,-83 1400,-83 1400,-83 1084,-83 1084,-83 1078,-83 1072,-77 1072,-71 1072,-71 1072,-20 1072,-20 1072,-14 1078,-8 1084,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"1242\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-52C132,-52 102,-52 102,-52 96,-52 90,-46 90,-40 90,-40 90,-28 90,-28 90,-22 96,-16 102,-16 102,-16 132,-16 132,-16 138,-16 144,-22 144,-28 144,-28 144,-40 144,-40 144,-46 138,-52 132,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn -->\n",
       "<g id=\"node2\" class=\"node\"><title>prep_bn</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-52C222,-52 192,-52 192,-52 186,-52 180,-46 180,-40 180,-40 180,-28 180,-28 180,-22 186,-16 192,-16 192,-16 222,-16 222,-16 228,-16 234,-22 234,-28 234,-28 234,-40 234,-40 234,-46 228,-52 222,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;prep_bn -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;prep_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-34C152.393,-34 161.311,-34 169.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-37.5001 179.919,-34 169.919,-30.5001 169.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- prep_relu -->\n",
       "<g id=\"node3\" class=\"node\"><title>prep_relu</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-52C312,-52 282,-52 282,-52 276,-52 270,-46 270,-40 270,-40 270,-28 270,-28 270,-22 276,-16 282,-16 282,-16 312,-16 312,-16 318,-16 324,-22 324,-28 324,-28 324,-40 324,-40 324,-46 318,-52 312,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn&#45;&gt;prep_relu -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>prep_bn&#45;&gt;prep_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-34C242.393,-34 251.311,-34 259.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-37.5001 269.919,-34 259.919,-30.5001 259.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_conv -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_conv</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-52C402,-52 372,-52 372,-52 366,-52 360,-46 360,-40 360,-40 360,-28 360,-28 360,-22 366,-16 372,-16 372,-16 402,-16 402,-16 408,-16 414,-22 414,-28 414,-28 414,-40 414,-40 414,-46 408,-52 402,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_relu&#45;&gt;layer1_conv -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>prep_relu&#45;&gt;layer1_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-34C332.393,-34 341.311,-34 349.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.919,-37.5001 359.919,-34 349.919,-30.5001 349.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_bn -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_bn</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-52C492,-52 462,-52 462,-52 456,-52 450,-46 450,-40 450,-40 450,-28 450,-28 450,-22 456,-16 462,-16 462,-16 492,-16 492,-16 498,-16 504,-22 504,-28 504,-28 504,-40 504,-40 504,-46 498,-52 492,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_conv&#45;&gt;layer1_bn -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_conv&#45;&gt;layer1_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-34C422.393,-34 431.311,-34 439.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-37.5001 449.919,-34 439.919,-30.5001 439.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_relu -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer1_relu</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-52C582,-52 552,-52 552,-52 546,-52 540,-46 540,-40 540,-40 540,-28 540,-28 540,-22 546,-16 552,-16 552,-16 582,-16 582,-16 588,-16 594,-22 594,-28 594,-28 594,-40 594,-40 594,-46 588,-52 582,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_bn&#45;&gt;layer1_relu -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_bn&#45;&gt;layer1_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-34C512.393,-34 521.311,-34 529.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-37.5001 539.919,-34 529.919,-30.5001 529.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_pool -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer1_pool</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M672,-52C672,-52 642,-52 642,-52 636,-52 630,-46 630,-40 630,-40 630,-28 630,-28 630,-22 636,-16 642,-16 642,-16 672,-16 672,-16 678,-16 684,-22 684,-28 684,-28 684,-40 684,-40 684,-46 678,-52 672,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_relu&#45;&gt;layer1_pool -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer1_relu&#45;&gt;layer1_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-34C602.393,-34 611.311,-34 619.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-37.5001 629.919,-34 619.919,-30.5001 619.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_conv -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer2_conv</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M762,-52C762,-52 732,-52 732,-52 726,-52 720,-46 720,-40 720,-40 720,-28 720,-28 720,-22 726,-16 732,-16 732,-16 762,-16 762,-16 768,-16 774,-22 774,-28 774,-28 774,-40 774,-40 774,-46 768,-52 762,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"747\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_pool&#45;&gt;layer2_conv -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer1_pool&#45;&gt;layer2_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.403,-34C692.393,-34 701.311,-34 709.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"709.919,-37.5001 719.919,-34 709.919,-30.5001 709.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_bn -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer2_bn</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M852,-52C852,-52 822,-52 822,-52 816,-52 810,-46 810,-40 810,-40 810,-28 810,-28 810,-22 816,-16 822,-16 822,-16 852,-16 852,-16 858,-16 864,-22 864,-28 864,-28 864,-40 864,-40 864,-46 858,-52 852,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_conv&#45;&gt;layer2_bn -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer2_conv&#45;&gt;layer2_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.403,-34C782.393,-34 791.311,-34 799.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"799.919,-37.5001 809.919,-34 799.919,-30.5001 799.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_relu -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer2_relu</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M942,-52C942,-52 912,-52 912,-52 906,-52 900,-46 900,-40 900,-40 900,-28 900,-28 900,-22 906,-16 912,-16 912,-16 942,-16 942,-16 948,-16 954,-22 954,-28 954,-28 954,-40 954,-40 954,-46 948,-52 942,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_bn&#45;&gt;layer2_relu -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer2_bn&#45;&gt;layer2_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.403,-34C872.393,-34 881.311,-34 889.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"889.919,-37.5001 899.919,-34 889.919,-30.5001 889.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_pool -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer2_pool</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1032,-52C1032,-52 1002,-52 1002,-52 996,-52 990,-46 990,-40 990,-40 990,-28 990,-28 990,-22 996,-16 1002,-16 1002,-16 1032,-16 1032,-16 1038,-16 1044,-22 1044,-28 1044,-28 1044,-40 1044,-40 1044,-46 1038,-52 1032,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_relu&#45;&gt;layer2_pool -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer2_relu&#45;&gt;layer2_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M954.403,-34C962.393,-34 971.311,-34 979.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"979.919,-37.5001 989.919,-34 979.919,-30.5001 979.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_conv -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer3_conv</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 512, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1122,-52C1122,-52 1092,-52 1092,-52 1086,-52 1080,-46 1080,-40 1080,-40 1080,-28 1080,-28 1080,-22 1086,-16 1092,-16 1092,-16 1122,-16 1122,-16 1128,-16 1134,-22 1134,-28 1134,-28 1134,-40 1134,-40 1134,-46 1128,-52 1122,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_pool&#45;&gt;layer3_conv -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer2_pool&#45;&gt;layer3_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.4,-34C1052.39,-34 1061.31,-34 1069.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1069.92,-37.5001 1079.92,-34 1069.92,-30.5001 1069.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_bn -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer3_bn</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 512, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1212,-52C1212,-52 1182,-52 1182,-52 1176,-52 1170,-46 1170,-40 1170,-40 1170,-28 1170,-28 1170,-22 1176,-16 1182,-16 1182,-16 1212,-16 1212,-16 1218,-16 1224,-22 1224,-28 1224,-28 1224,-40 1224,-40 1224,-46 1218,-52 1212,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_conv&#45;&gt;layer3_bn -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer3_conv&#45;&gt;layer3_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1134.4,-34C1142.39,-34 1151.31,-34 1159.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1159.92,-37.5001 1169.92,-34 1159.92,-30.5001 1159.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_relu -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer3_relu</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1302,-52C1302,-52 1272,-52 1272,-52 1266,-52 1260,-46 1260,-40 1260,-40 1260,-28 1260,-28 1260,-22 1266,-16 1272,-16 1272,-16 1302,-16 1302,-16 1308,-16 1314,-22 1314,-28 1314,-28 1314,-40 1314,-40 1314,-46 1308,-52 1302,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1287\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_bn&#45;&gt;layer3_relu -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer3_bn&#45;&gt;layer3_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1224.4,-34C1232.39,-34 1241.31,-34 1249.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1249.92,-37.5001 1259.92,-34 1249.92,-30.5001 1249.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_pool -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer3_pool</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1392,-52C1392,-52 1362,-52 1362,-52 1356,-52 1350,-46 1350,-40 1350,-40 1350,-28 1350,-28 1350,-22 1356,-16 1362,-16 1362,-16 1392,-16 1392,-16 1398,-16 1404,-22 1404,-28 1404,-28 1404,-40 1404,-40 1404,-46 1398,-52 1392,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1377\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_relu&#45;&gt;layer3_pool -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer3_relu&#45;&gt;layer3_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1314.4,-34C1322.39,-34 1331.31,-34 1339.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1339.92,-37.5001 1349.92,-34 1339.92,-30.5001 1339.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- pool -->\n",
       "<g id=\"node16\" class=\"node\"><title>pool</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1482,-52C1482,-52 1452,-52 1452,-52 1446,-52 1440,-46 1440,-40 1440,-40 1440,-28 1440,-28 1440,-22 1446,-16 1452,-16 1452,-16 1482,-16 1482,-16 1488,-16 1494,-22 1494,-28 1494,-28 1494,-40 1494,-40 1494,-46 1488,-52 1482,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1467\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_pool&#45;&gt;pool -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer3_pool&#45;&gt;pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1404.4,-34C1412.39,-34 1421.31,-34 1429.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1429.92,-37.5001 1439.92,-34 1429.92,-30.5001 1429.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- flatten -->\n",
       "<g id=\"node17\" class=\"node\"><title>flatten</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M1572,-52C1572,-52 1542,-52 1542,-52 1536,-52 1530,-46 1530,-40 1530,-40 1530,-28 1530,-28 1530,-22 1536,-16 1542,-16 1542,-16 1572,-16 1572,-16 1578,-16 1584,-22 1584,-28 1584,-28 1584,-40 1584,-40 1584,-46 1578,-52 1572,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1557\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pool&#45;&gt;flatten -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>pool&#45;&gt;flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1494.4,-34C1502.39,-34 1511.31,-34 1519.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1519.92,-37.5001 1529.92,-34 1519.92,-30.5001 1519.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- linear -->\n",
       "<g id=\"node18\" class=\"node\"><title>linear</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M1662,-52C1662,-52 1632,-52 1632,-52 1626,-52 1620,-46 1620,-40 1620,-40 1620,-28 1620,-28 1620,-22 1626,-16 1632,-16 1632,-16 1662,-16 1662,-16 1668,-16 1674,-22 1674,-28 1674,-28 1674,-40 1674,-40 1674,-46 1668,-52 1662,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1647\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- flatten&#45;&gt;linear -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>flatten&#45;&gt;linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1584.4,-34C1592.39,-34 1601.31,-34 1609.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1609.92,-37.5001 1619.92,-34 1609.92,-30.5001 1609.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node19\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"Mul {&#39;weight&#39;: 0.125}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M1763,-52C1763,-52 1722,-52 1722,-52 1716,-52 1710,-46 1710,-40 1710,-40 1710,-28 1710,-28 1710,-22 1716,-16 1722,-16 1722,-16 1763,-16 1763,-16 1769,-16 1775,-22 1775,-28 1775,-28 1775,-40 1775,-40 1775,-46 1769,-52 1763,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1742.5\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- linear&#45;&gt;classifier -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1674.04,-34C1682.05,-34 1691.06,-34 1699.83,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1699.95,-37.5001 1709.94,-34 1699.94,-30.5001 1699.95,-37.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node20\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-52C42,-52 12,-52 12,-52 6,-52 0,-46 0,-40 0,-40 0,-28 0,-28 0,-22 6,-16 12,-16 12,-16 42,-16 42,-16 48,-16 54,-22 54,-28 54,-28 54,-40 54,-40 54,-46 48,-52 42,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-34C62.3932,-34 71.3106,-34 79.8241,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-37.5001 89.919,-34 79.919,-30.5001 79.919,-37.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7fcd233ee8d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.1000       2.0612       1.7973       0.3648       0.1531       1.2866       0.5386       2.2143\n",
      "           2       0.2000       2.1240       1.1342       0.5999       0.1456       0.9052       0.6766       4.4839\n",
      "           3       0.3000       2.0614       0.8749       0.6971       0.1443       0.8934       0.6953       6.6896\n",
      "           4       0.4000       2.0633       0.7480       0.7435       0.1631       0.7204       0.7483       8.9161\n",
      "           5       0.3750       2.0654       0.6618       0.7746       0.1521       0.6767       0.7770      11.1336\n",
      "           6       0.3500       2.0963       0.5849       0.8033       0.1444       0.6356       0.7864      13.3742\n",
      "           7       0.3250       2.0576       0.5370       0.8225       0.1535       0.5014       0.8319      15.5853\n",
      "           8       0.3000       2.0629       0.5013       0.8329       0.1532       0.5020       0.8335      17.8014\n",
      "           9       0.2750       2.0624       0.4752       0.8440       0.1443       0.6030       0.7962      20.0081\n",
      "          10       0.2500       2.1048       0.4503       0.8516       0.1507       0.5633       0.8081      22.2636\n",
      "          11       0.2250       2.0608       0.4281       0.8585       0.1513       0.4647       0.8386      24.4757\n",
      "          12       0.2000       2.0601       0.4062       0.8686       0.1574       0.4031       0.8610      26.6932\n",
      "          13       0.1750       2.0612       0.3880       0.8743       0.1550       0.4340       0.8565      28.9094\n",
      "          14       0.1500       2.1139       0.3649       0.8822       0.1539       0.4364       0.8515      31.1772\n",
      "          15       0.1250       2.0591       0.3437       0.8895       0.1526       0.3603       0.8778      33.3889\n",
      "          16       0.1000       2.0596       0.3241       0.8966       0.1549       0.3967       0.8661      35.6034\n",
      "          17       0.0750       2.0586       0.3005       0.9049       0.1537       0.3129       0.8981      37.8157\n",
      "          18       0.0500       2.1106       0.2750       0.9142       0.1483       0.3047       0.8961      40.0746\n",
      "          19       0.0250       2.0550       0.2531       0.9223       0.1505       0.2934       0.9043      42.2801\n",
      "          20       0.0000       2.0592       0.2284       0.9316       0.1561       0.2660       0.9107      44.4955\n"
     ]
    }
   ],
   "source": [
    "lr_schedule = PiecewiseLinear([0, 4, 20], [0, 0.4, 0])\n",
    "batch_size = 512\n",
    "\n",
    "n = net(extra_layers=(), res_layers=())\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(8,8)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 4: Architecture](https://www.myrtle.ai/2018/10/26/how_to_train_your_resnet_4/)  - double width, 60 epoch train! (321s; test acc 93.5%)\n",
    "\n",
    "ne approach that doesn't seem particularly promising is to just add width. \n",
    "\n",
    "If we double the channel dimensions and train for 60 epochs we can reach 93.5% test accuracy with a 5 layer network. This is nice but not efficient since training now takes 321s."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"60pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 59.97\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.605721 0.605721) rotate(0) translate(4 95)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-95 1779,-95 1779,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-8C94,-8 320,-8 320,-8 326,-8 332,-14 332,-20 332,-20 332,-71 332,-71 332,-77 326,-83 320,-83 320,-83 94,-83 94,-83 88,-83 82,-77 82,-71 82,-71 82,-20 82,-20 82,-14 88,-8 94,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-8C364,-8 680,-8 680,-8 686,-8 692,-14 692,-20 692,-20 692,-71 692,-71 692,-77 686,-83 680,-83 680,-83 364,-83 364,-83 358,-83 352,-77 352,-71 352,-71 352,-20 352,-20 352,-14 358,-8 364,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"522\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M724,-8C724,-8 1040,-8 1040,-8 1046,-8 1052,-14 1052,-20 1052,-20 1052,-71 1052,-71 1052,-77 1046,-83 1040,-83 1040,-83 724,-83 724,-83 718,-83 712,-77 712,-71 712,-71 712,-20 712,-20 712,-14 718,-8 724,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"882\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1084,-8C1084,-8 1400,-8 1400,-8 1406,-8 1412,-14 1412,-20 1412,-20 1412,-71 1412,-71 1412,-77 1406,-83 1400,-83 1400,-83 1084,-83 1084,-83 1078,-83 1072,-77 1072,-71 1072,-71 1072,-20 1072,-20 1072,-14 1078,-8 1084,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"1242\" y=\"-67.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-52C132,-52 102,-52 102,-52 96,-52 90,-46 90,-40 90,-40 90,-28 90,-28 90,-22 96,-16 102,-16 102,-16 132,-16 132,-16 138,-16 144,-22 144,-28 144,-28 144,-40 144,-40 144,-46 138,-52 132,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn -->\n",
       "<g id=\"node2\" class=\"node\"><title>prep_bn</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-52C222,-52 192,-52 192,-52 186,-52 180,-46 180,-40 180,-40 180,-28 180,-28 180,-22 186,-16 192,-16 192,-16 222,-16 222,-16 228,-16 234,-22 234,-28 234,-28 234,-40 234,-40 234,-46 228,-52 222,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;prep_bn -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;prep_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-34C152.393,-34 161.311,-34 169.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-37.5001 179.919,-34 169.919,-30.5001 169.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- prep_relu -->\n",
       "<g id=\"node3\" class=\"node\"><title>prep_relu</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-52C312,-52 282,-52 282,-52 276,-52 270,-46 270,-40 270,-40 270,-28 270,-28 270,-22 276,-16 282,-16 282,-16 312,-16 312,-16 318,-16 324,-22 324,-28 324,-28 324,-40 324,-40 324,-46 318,-52 312,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn&#45;&gt;prep_relu -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>prep_bn&#45;&gt;prep_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-34C242.393,-34 251.311,-34 259.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-37.5001 269.919,-34 259.919,-30.5001 259.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_conv -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_conv</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-52C402,-52 372,-52 372,-52 366,-52 360,-46 360,-40 360,-40 360,-28 360,-28 360,-22 366,-16 372,-16 372,-16 402,-16 402,-16 408,-16 414,-22 414,-28 414,-28 414,-40 414,-40 414,-46 408,-52 402,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_relu&#45;&gt;layer1_conv -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>prep_relu&#45;&gt;layer1_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-34C332.393,-34 341.311,-34 349.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.919,-37.5001 359.919,-34 349.919,-30.5001 349.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_bn -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_bn</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-52C492,-52 462,-52 462,-52 456,-52 450,-46 450,-40 450,-40 450,-28 450,-28 450,-22 456,-16 462,-16 462,-16 492,-16 492,-16 498,-16 504,-22 504,-28 504,-28 504,-40 504,-40 504,-46 498,-52 492,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_conv&#45;&gt;layer1_bn -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_conv&#45;&gt;layer1_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-34C422.393,-34 431.311,-34 439.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-37.5001 449.919,-34 439.919,-30.5001 439.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_relu -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer1_relu</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-52C582,-52 552,-52 552,-52 546,-52 540,-46 540,-40 540,-40 540,-28 540,-28 540,-22 546,-16 552,-16 552,-16 582,-16 582,-16 588,-16 594,-22 594,-28 594,-28 594,-40 594,-40 594,-46 588,-52 582,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_bn&#45;&gt;layer1_relu -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_bn&#45;&gt;layer1_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-34C512.393,-34 521.311,-34 529.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-37.5001 539.919,-34 529.919,-30.5001 529.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_pool -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer1_pool</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M672,-52C672,-52 642,-52 642,-52 636,-52 630,-46 630,-40 630,-40 630,-28 630,-28 630,-22 636,-16 642,-16 642,-16 672,-16 672,-16 678,-16 684,-22 684,-28 684,-28 684,-40 684,-40 684,-46 678,-52 672,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_relu&#45;&gt;layer1_pool -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer1_relu&#45;&gt;layer1_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-34C602.393,-34 611.311,-34 619.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-37.5001 629.919,-34 619.919,-30.5001 619.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_conv -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer2_conv</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 512, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M762,-52C762,-52 732,-52 732,-52 726,-52 720,-46 720,-40 720,-40 720,-28 720,-28 720,-22 726,-16 732,-16 732,-16 762,-16 762,-16 768,-16 774,-22 774,-28 774,-28 774,-40 774,-40 774,-46 768,-52 762,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"747\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_pool&#45;&gt;layer2_conv -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer1_pool&#45;&gt;layer2_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.403,-34C692.393,-34 701.311,-34 709.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"709.919,-37.5001 719.919,-34 709.919,-30.5001 709.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_bn -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer2_bn</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 512, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M852,-52C852,-52 822,-52 822,-52 816,-52 810,-46 810,-40 810,-40 810,-28 810,-28 810,-22 816,-16 822,-16 822,-16 852,-16 852,-16 858,-16 864,-22 864,-28 864,-28 864,-40 864,-40 864,-46 858,-52 852,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_conv&#45;&gt;layer2_bn -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer2_conv&#45;&gt;layer2_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.403,-34C782.393,-34 791.311,-34 799.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"799.919,-37.5001 809.919,-34 799.919,-30.5001 799.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_relu -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer2_relu</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M942,-52C942,-52 912,-52 912,-52 906,-52 900,-46 900,-40 900,-40 900,-28 900,-28 900,-22 906,-16 912,-16 912,-16 942,-16 942,-16 948,-16 954,-22 954,-28 954,-28 954,-40 954,-40 954,-46 948,-52 942,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_bn&#45;&gt;layer2_relu -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer2_bn&#45;&gt;layer2_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.403,-34C872.393,-34 881.311,-34 889.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"889.919,-37.5001 899.919,-34 889.919,-30.5001 889.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_pool -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer2_pool</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1032,-52C1032,-52 1002,-52 1002,-52 996,-52 990,-46 990,-40 990,-40 990,-28 990,-28 990,-22 996,-16 1002,-16 1002,-16 1032,-16 1032,-16 1038,-16 1044,-22 1044,-28 1044,-28 1044,-40 1044,-40 1044,-46 1038,-52 1032,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_relu&#45;&gt;layer2_pool -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer2_relu&#45;&gt;layer2_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M954.403,-34C962.393,-34 971.311,-34 979.824,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"979.919,-37.5001 989.919,-34 979.919,-30.5001 979.919,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_conv -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer3_conv</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 512, &#39;out_channels&#39;: 1024, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1122,-52C1122,-52 1092,-52 1092,-52 1086,-52 1080,-46 1080,-40 1080,-40 1080,-28 1080,-28 1080,-22 1086,-16 1092,-16 1092,-16 1122,-16 1122,-16 1128,-16 1134,-22 1134,-28 1134,-28 1134,-40 1134,-40 1134,-46 1128,-52 1122,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_pool&#45;&gt;layer3_conv -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer2_pool&#45;&gt;layer3_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.4,-34C1052.39,-34 1061.31,-34 1069.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1069.92,-37.5001 1079.92,-34 1069.92,-30.5001 1069.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_bn -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer3_bn</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 1024, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1212,-52C1212,-52 1182,-52 1182,-52 1176,-52 1170,-46 1170,-40 1170,-40 1170,-28 1170,-28 1170,-22 1176,-16 1182,-16 1182,-16 1212,-16 1212,-16 1218,-16 1224,-22 1224,-28 1224,-28 1224,-40 1224,-40 1224,-46 1218,-52 1212,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_conv&#45;&gt;layer3_bn -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer3_conv&#45;&gt;layer3_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1134.4,-34C1142.39,-34 1151.31,-34 1159.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1159.92,-37.5001 1169.92,-34 1159.92,-30.5001 1159.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_relu -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer3_relu</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1302,-52C1302,-52 1272,-52 1272,-52 1266,-52 1260,-46 1260,-40 1260,-40 1260,-28 1260,-28 1260,-22 1266,-16 1272,-16 1272,-16 1302,-16 1302,-16 1308,-16 1314,-22 1314,-28 1314,-28 1314,-40 1314,-40 1314,-46 1308,-52 1302,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1287\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_bn&#45;&gt;layer3_relu -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer3_bn&#45;&gt;layer3_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1224.4,-34C1232.39,-34 1241.31,-34 1249.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1249.92,-37.5001 1259.92,-34 1249.92,-30.5001 1249.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_pool -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer3_pool</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1392,-52C1392,-52 1362,-52 1362,-52 1356,-52 1350,-46 1350,-40 1350,-40 1350,-28 1350,-28 1350,-22 1356,-16 1362,-16 1362,-16 1392,-16 1392,-16 1398,-16 1404,-22 1404,-28 1404,-28 1404,-40 1404,-40 1404,-46 1398,-52 1392,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1377\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_relu&#45;&gt;layer3_pool -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer3_relu&#45;&gt;layer3_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1314.4,-34C1322.39,-34 1331.31,-34 1339.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1339.92,-37.5001 1349.92,-34 1339.92,-30.5001 1339.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- pool -->\n",
       "<g id=\"node16\" class=\"node\"><title>pool</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1482,-52C1482,-52 1452,-52 1452,-52 1446,-52 1440,-46 1440,-40 1440,-40 1440,-28 1440,-28 1440,-22 1446,-16 1452,-16 1452,-16 1482,-16 1482,-16 1488,-16 1494,-22 1494,-28 1494,-28 1494,-40 1494,-40 1494,-46 1488,-52 1482,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1467\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_pool&#45;&gt;pool -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer3_pool&#45;&gt;pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1404.4,-34C1412.39,-34 1421.31,-34 1429.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1429.92,-37.5001 1439.92,-34 1429.92,-30.5001 1429.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- flatten -->\n",
       "<g id=\"node17\" class=\"node\"><title>flatten</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M1572,-52C1572,-52 1542,-52 1542,-52 1536,-52 1530,-46 1530,-40 1530,-40 1530,-28 1530,-28 1530,-22 1536,-16 1542,-16 1542,-16 1572,-16 1572,-16 1578,-16 1584,-22 1584,-28 1584,-28 1584,-40 1584,-40 1584,-46 1578,-52 1572,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1557\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pool&#45;&gt;flatten -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>pool&#45;&gt;flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1494.4,-34C1502.39,-34 1511.31,-34 1519.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1519.92,-37.5001 1529.92,-34 1519.92,-30.5001 1519.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- linear -->\n",
       "<g id=\"node18\" class=\"node\"><title>linear</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Linear {&#39;in_features&#39;: 1024, &#39;out_features&#39;: 10, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M1662,-52C1662,-52 1632,-52 1632,-52 1626,-52 1620,-46 1620,-40 1620,-40 1620,-28 1620,-28 1620,-22 1626,-16 1632,-16 1632,-16 1662,-16 1662,-16 1668,-16 1674,-22 1674,-28 1674,-28 1674,-40 1674,-40 1674,-46 1668,-52 1662,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1647\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- flatten&#45;&gt;linear -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>flatten&#45;&gt;linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1584.4,-34C1592.39,-34 1601.31,-34 1609.82,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1609.92,-37.5001 1619.92,-34 1609.92,-30.5001 1609.92,-37.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node19\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"Mul {&#39;weight&#39;: 0.125}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M1763,-52C1763,-52 1722,-52 1722,-52 1716,-52 1710,-46 1710,-40 1710,-40 1710,-28 1710,-28 1710,-22 1716,-16 1722,-16 1722,-16 1763,-16 1763,-16 1769,-16 1775,-22 1775,-28 1775,-28 1775,-40 1775,-40 1775,-46 1769,-52 1763,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"1742.5\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- linear&#45;&gt;classifier -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1674.04,-34C1682.05,-34 1691.06,-34 1699.83,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1699.95,-37.5001 1709.94,-34 1699.94,-30.5001 1699.95,-37.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node20\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-52C42,-52 12,-52 12,-52 6,-52 0,-46 0,-40 0,-40 0,-28 0,-28 0,-22 6,-16 12,-16 12,-16 42,-16 42,-16 48,-16 54,-22 54,-28 54,-28 54,-40 54,-40 54,-46 48,-52 42,-52\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-34C62.3932,-34 71.3106,-34 79.8241,-34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-37.5001 89.919,-34 79.919,-30.5001 79.919,-37.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7fcd233e0080>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.0333       9.6687       1.8879       0.3291       0.7929       1.6214       0.4464      10.4616\n",
      "           2       0.0667       5.3167       1.2460       0.5628       0.3784       1.0503       0.6263      16.1567\n",
      "           3       0.1000       5.3180       0.9350       0.6807       0.3807       0.7622       0.7368      21.8554\n",
      "           4       0.1333       5.3191       0.7673       0.7385       0.3788       0.7431       0.7427      27.5533\n",
      "           5       0.1667       5.3328       0.6730       0.7719       0.3793       0.7541       0.7432      33.2654\n",
      "           6       0.2000       5.3204       0.6109       0.7939       0.3790       0.5805       0.8037      38.9648\n",
      "           7       0.2333       5.3206       0.5681       0.8102       0.3781       0.5228       0.8300      44.6635\n",
      "           8       0.2667       5.3186       0.5294       0.8226       0.3786       0.5259       0.8180      50.3607\n",
      "           9       0.3000       5.3371       0.5010       0.8337       0.3786       0.5477       0.8130      56.0763\n",
      "          10       0.3333       5.3160       0.4788       0.8409       0.3773       0.5411       0.8169      61.7696\n",
      "          11       0.3667       5.3110       0.4667       0.8449       0.3770       0.5272       0.8170      67.4575\n",
      "          12       0.4000       5.3065       0.4615       0.8483       0.3761       0.5146       0.8277      73.1401\n",
      "          13       0.3917       5.3185       0.4619       0.8468       0.3759       0.5915       0.7921      78.8345\n",
      "          14       0.3833       5.2988       0.4328       0.8576       0.3760       0.6254       0.7810      84.5093\n",
      "          15       0.3750       5.2988       0.4200       0.8614       0.3762       0.4879       0.8336      90.1844\n",
      "          16       0.3667       5.2954       0.4091       0.8654       0.3762       0.5231       0.8253      95.8559\n",
      "          17       0.3583       5.3100       0.4012       0.8694       0.3759       0.4313       0.8570     101.5418\n",
      "          18       0.3500       5.2874       0.3893       0.8735       0.3751       0.5420       0.8216     107.2044\n",
      "          19       0.3417       5.2837       0.3812       0.8757       0.3753       0.4562       0.8528     112.8633\n",
      "          20       0.3333       5.2845       0.3776       0.8773       0.3766       0.5398       0.8145     118.5244\n",
      "          21       0.3250       5.3068       0.3648       0.8803       0.3756       0.5970       0.7918     124.2069\n",
      "          22       0.3167       5.2856       0.3599       0.8831       0.3757       0.3785       0.8741     129.8681\n",
      "          23       0.3083       5.2844       0.3502       0.8861       0.3752       0.4008       0.8661     135.5278\n",
      "          24       0.3000       5.2836       0.3470       0.8872       0.3756       0.4075       0.8660     141.1870\n",
      "          25       0.2917       5.2997       0.3375       0.8921       0.3750       0.4506       0.8522     146.8617\n",
      "          26       0.2833       5.2832       0.3295       0.8943       0.3755       0.4160       0.8564     152.5203\n",
      "          27       0.2750       5.2817       0.3242       0.8960       0.3752       0.3765       0.8703     158.1772\n",
      "          28       0.2667       5.2830       0.3202       0.8972       0.3755       0.3968       0.8673     163.8357\n",
      "          29       0.2583       5.3022       0.3168       0.8986       0.3749       0.4164       0.8591     169.5128\n",
      "          30       0.2500       5.2820       0.3119       0.9003       0.3758       0.3893       0.8708     175.1706\n",
      "          31       0.2417       5.2828       0.3005       0.9034       0.3749       0.3858       0.8695     180.8283\n",
      "          32       0.2333       5.2835       0.2927       0.9071       0.3749       0.4106       0.8669     186.4866\n",
      "          33       0.2250       5.2807       0.2833       0.9095       0.4139       0.4275       0.8503     192.1813\n",
      "          34       0.2167       5.2809       0.2836       0.9092       0.3753       0.3662       0.8733     197.8374\n",
      "          35       0.2083       5.2798       0.2724       0.9146       0.3756       0.3642       0.8792     203.4929\n",
      "          36       0.2000       5.2739       0.2663       0.9156       0.3747       0.3594       0.8774     209.1415\n",
      "          37       0.1917       5.2786       0.2614       0.9182       0.4146       0.3657       0.8753     214.8347\n",
      "          38       0.1833       5.2794       0.2560       0.9194       0.3750       0.3412       0.8840     220.4891\n",
      "          39       0.1750       5.2778       0.2518       0.9213       0.3743       0.3679       0.8766     226.1412\n",
      "          40       0.1667       5.2759       0.2449       0.9231       0.3743       0.3205       0.8906     231.7915\n",
      "          41       0.1583       5.2797       0.2348       0.9279       0.4033       0.3754       0.8695     237.4745\n",
      "          42       0.1500       5.2731       0.2265       0.9306       0.3748       0.3243       0.8942     243.1224\n",
      "          43       0.1417       5.2753       0.2174       0.9347       0.3744       0.3599       0.8804     248.7721\n",
      "          44       0.1333       5.2812       0.2142       0.9343       0.3743       0.3088       0.8990     254.4276\n",
      "          45       0.1250       5.2714       0.2005       0.9400       0.3987       0.3233       0.8911     260.0976\n",
      "          46       0.1167       5.2738       0.1957       0.9415       0.3744       0.3042       0.9002     265.7459\n",
      "          47       0.1083       5.2768       0.1855       0.9447       0.3742       0.2897       0.9020     271.3969\n",
      "          48       0.1000       5.2762       0.1795       0.9477       0.3744       0.3164       0.8933     277.0475\n",
      "          49       0.0917       5.2797       0.1693       0.9526       0.4178       0.2789       0.9016     282.7449\n",
      "          50       0.0833       5.2773       0.1610       0.9542       0.3739       0.2676       0.9096     288.3961\n",
      "          51       0.0750       5.2756       0.1488       0.9593       0.3744       0.2636       0.9123     294.0462\n",
      "          52       0.0667       5.2770       0.1412       0.9613       0.3744       0.2697       0.9076     299.6976\n",
      "          53       0.0583       5.2790       0.1321       0.9643       0.3740       0.2663       0.9076     305.3505\n",
      "          54       0.0500       5.2896       0.1196       0.9689       0.3747       0.2341       0.9198     311.0148\n",
      "          55       0.0417       5.2747       0.1106       0.9728       0.3744       0.2398       0.9178     316.6639\n",
      "          56       0.0333       5.2711       0.1012       0.9752       0.3739       0.2187       0.9283     322.3089\n",
      "          57       0.0250       5.2725       0.0904       0.9796       0.3736       0.2109       0.9283     327.9550\n",
      "          58       0.0167       5.2852       0.0815       0.9824       0.3741       0.2101       0.9307     333.6143\n",
      "          59       0.0083       5.2686       0.0759       0.9843       0.3740       0.2009       0.9332     339.2569\n",
      "          60       0.0000       5.2742       0.0690       0.9863       0.3740       0.2001       0.9329     344.9050\n"
     ]
    }
   ],
   "source": [
    "lr_schedule = PiecewiseLinear([0, 12, 60], [0, 0.4, 0])\n",
    "batch_size = 512\n",
    "c = 128\n",
    "\n",
    "n = net(channels={'prep': c, 'layer1': 2*c, 'layer2': 4*c, 'layer3': 8*c}, extra_layers=(), res_layers=())\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(8,8)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 4: Architecture](https://www.myrtle.ai/2018/10/26/how_to_train_your_resnet_4/)  - extra:L1+L2+L3 network, 60 epochs, cutout=12 (180s, 95.0% test acc) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"57pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 57.48\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.416506 0.416506) rotate(0) translate(4 134)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-134 2589,-134 2589,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-16C94,-16 320,-16 320,-16 326,-16 332,-22 332,-28 332,-28 332,-79 332,-79 332,-85 326,-91 320,-91 320,-91 94,-91 94,-91 88,-91 82,-85 82,-79 82,-79 82,-28 82,-28 82,-22 88,-16 94,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-8C364,-8 958,-8 958,-8 964,-8 970,-14 970,-20 970,-20 970,-110 970,-110 970,-116 964,-122 958,-122 958,-122 364,-122 364,-122 358,-122 352,-116 352,-110 352,-110 352,-20 352,-20 352,-14 358,-8 364,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"661\" y=\"-106.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer1_extra</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M724,-16C724,-16 950,-16 950,-16 956,-16 962,-22 962,-28 962,-28 962,-79 962,-79 962,-85 956,-91 950,-91 950,-91 724,-91 724,-91 718,-91 712,-85 712,-79 712,-79 712,-28 712,-28 712,-22 718,-16 724,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">extra</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M994,-8C994,-8 1588,-8 1588,-8 1594,-8 1600,-14 1600,-20 1600,-20 1600,-110 1600,-110 1600,-116 1594,-122 1588,-122 1588,-122 994,-122 994,-122 988,-122 982,-116 982,-110 982,-110 982,-20 982,-20 982,-14 988,-8 994,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"1291\" y=\"-106.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer2_extra</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1354,-16C1354,-16 1580,-16 1580,-16 1586,-16 1592,-22 1592,-28 1592,-28 1592,-79 1592,-79 1592,-85 1586,-91 1580,-91 1580,-91 1354,-91 1354,-91 1348,-91 1342,-85 1342,-79 1342,-79 1342,-28 1342,-28 1342,-22 1348,-16 1354,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"1467\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">extra</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1624,-8C1624,-8 2218,-8 2218,-8 2224,-8 2230,-14 2230,-20 2230,-20 2230,-110 2230,-110 2230,-116 2224,-122 2218,-122 2218,-122 1624,-122 1624,-122 1618,-122 1612,-116 1612,-110 1612,-110 1612,-20 1612,-20 1612,-14 1618,-8 1624,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921\" y=\"-106.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer3_extra</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1984,-16C1984,-16 2210,-16 2210,-16 2216,-16 2222,-22 2222,-28 2222,-28 2222,-79 2222,-79 2222,-85 2216,-91 2210,-91 2210,-91 1984,-91 1984,-91 1978,-91 1972,-85 1972,-79 1972,-79 1972,-28 1972,-28 1972,-22 1978,-16 1984,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"2097\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">extra</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-60C132,-60 102,-60 102,-60 96,-60 90,-54 90,-48 90,-48 90,-36 90,-36 90,-30 96,-24 102,-24 102,-24 132,-24 132,-24 138,-24 144,-30 144,-36 144,-36 144,-48 144,-48 144,-54 138,-60 132,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn -->\n",
       "<g id=\"node2\" class=\"node\"><title>prep_bn</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-60C222,-60 192,-60 192,-60 186,-60 180,-54 180,-48 180,-48 180,-36 180,-36 180,-30 186,-24 192,-24 192,-24 222,-24 222,-24 228,-24 234,-30 234,-36 234,-36 234,-48 234,-48 234,-54 228,-60 222,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;prep_bn -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;prep_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-42C152.393,-42 161.311,-42 169.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-45.5001 179.919,-42 169.919,-38.5001 169.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- prep_relu -->\n",
       "<g id=\"node3\" class=\"node\"><title>prep_relu</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-60C312,-60 282,-60 282,-60 276,-60 270,-54 270,-48 270,-48 270,-36 270,-36 270,-30 276,-24 282,-24 282,-24 312,-24 312,-24 318,-24 324,-30 324,-36 324,-36 324,-48 324,-48 324,-54 318,-60 312,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn&#45;&gt;prep_relu -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>prep_bn&#45;&gt;prep_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-42C242.393,-42 251.311,-42 259.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-45.5001 269.919,-42 259.919,-38.5001 259.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_conv -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_conv</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-60C402,-60 372,-60 372,-60 366,-60 360,-54 360,-48 360,-48 360,-36 360,-36 360,-30 366,-24 372,-24 372,-24 402,-24 402,-24 408,-24 414,-30 414,-36 414,-36 414,-48 414,-48 414,-54 408,-60 402,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_relu&#45;&gt;layer1_conv -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>prep_relu&#45;&gt;layer1_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-42C332.393,-42 341.311,-42 349.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.919,-45.5001 359.919,-42 349.919,-38.5001 349.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_bn -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_bn</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-60C492,-60 462,-60 462,-60 456,-60 450,-54 450,-48 450,-48 450,-36 450,-36 450,-30 456,-24 462,-24 462,-24 492,-24 492,-24 498,-24 504,-30 504,-36 504,-36 504,-48 504,-48 504,-54 498,-60 492,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_conv&#45;&gt;layer1_bn -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_conv&#45;&gt;layer1_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-42C422.393,-42 431.311,-42 439.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-45.5001 449.919,-42 439.919,-38.5001 439.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_relu -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer1_relu</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-60C582,-60 552,-60 552,-60 546,-60 540,-54 540,-48 540,-48 540,-36 540,-36 540,-30 546,-24 552,-24 552,-24 582,-24 582,-24 588,-24 594,-30 594,-36 594,-36 594,-48 594,-48 594,-54 588,-60 582,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_bn&#45;&gt;layer1_relu -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_bn&#45;&gt;layer1_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-42C512.393,-42 521.311,-42 529.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-45.5001 539.919,-42 529.919,-38.5001 529.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_pool -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer1_pool</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M672,-60C672,-60 642,-60 642,-60 636,-60 630,-54 630,-48 630,-48 630,-36 630,-36 630,-30 636,-24 642,-24 642,-24 672,-24 672,-24 678,-24 684,-30 684,-36 684,-36 684,-48 684,-48 684,-54 678,-60 672,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_relu&#45;&gt;layer1_pool -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer1_relu&#45;&gt;layer1_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-42C602.393,-42 611.311,-42 619.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-45.5001 629.919,-42 619.919,-38.5001 619.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_extra_conv -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer1_extra_conv</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M762,-60C762,-60 732,-60 732,-60 726,-60 720,-54 720,-48 720,-48 720,-36 720,-36 720,-30 726,-24 732,-24 732,-24 762,-24 762,-24 768,-24 774,-30 774,-36 774,-36 774,-48 774,-48 774,-54 768,-60 762,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"747\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_pool&#45;&gt;layer1_extra_conv -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer1_pool&#45;&gt;layer1_extra_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.403,-42C692.393,-42 701.311,-42 709.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"709.919,-45.5001 719.919,-42 709.919,-38.5001 709.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_extra_bn -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer1_extra_bn</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M852,-60C852,-60 822,-60 822,-60 816,-60 810,-54 810,-48 810,-48 810,-36 810,-36 810,-30 816,-24 822,-24 822,-24 852,-24 852,-24 858,-24 864,-30 864,-36 864,-36 864,-48 864,-48 864,-54 858,-60 852,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_extra_conv&#45;&gt;layer1_extra_bn -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer1_extra_conv&#45;&gt;layer1_extra_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.403,-42C782.393,-42 791.311,-42 799.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"799.919,-45.5001 809.919,-42 799.919,-38.5001 799.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_extra_relu -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer1_extra_relu</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M942,-60C942,-60 912,-60 912,-60 906,-60 900,-54 900,-48 900,-48 900,-36 900,-36 900,-30 906,-24 912,-24 912,-24 942,-24 942,-24 948,-24 954,-30 954,-36 954,-36 954,-48 954,-48 954,-54 948,-60 942,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_extra_bn&#45;&gt;layer1_extra_relu -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer1_extra_bn&#45;&gt;layer1_extra_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.403,-42C872.393,-42 881.311,-42 889.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"889.919,-45.5001 899.919,-42 889.919,-38.5001 889.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_conv -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer2_conv</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1032,-60C1032,-60 1002,-60 1002,-60 996,-60 990,-54 990,-48 990,-48 990,-36 990,-36 990,-30 996,-24 1002,-24 1002,-24 1032,-24 1032,-24 1038,-24 1044,-30 1044,-36 1044,-36 1044,-48 1044,-48 1044,-54 1038,-60 1032,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_extra_relu&#45;&gt;layer2_conv -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer1_extra_relu&#45;&gt;layer2_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M954.403,-42C962.393,-42 971.311,-42 979.824,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"979.919,-45.5001 989.919,-42 979.919,-38.5001 979.919,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_bn -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer2_bn</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1122,-60C1122,-60 1092,-60 1092,-60 1086,-60 1080,-54 1080,-48 1080,-48 1080,-36 1080,-36 1080,-30 1086,-24 1092,-24 1092,-24 1122,-24 1122,-24 1128,-24 1134,-30 1134,-36 1134,-36 1134,-48 1134,-48 1134,-54 1128,-60 1122,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_conv&#45;&gt;layer2_bn -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer2_conv&#45;&gt;layer2_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.4,-42C1052.39,-42 1061.31,-42 1069.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1069.92,-45.5001 1079.92,-42 1069.92,-38.5001 1069.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_relu -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer2_relu</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1212,-60C1212,-60 1182,-60 1182,-60 1176,-60 1170,-54 1170,-48 1170,-48 1170,-36 1170,-36 1170,-30 1176,-24 1182,-24 1182,-24 1212,-24 1212,-24 1218,-24 1224,-30 1224,-36 1224,-36 1224,-48 1224,-48 1224,-54 1218,-60 1212,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_bn&#45;&gt;layer2_relu -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer2_bn&#45;&gt;layer2_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1134.4,-42C1142.39,-42 1151.31,-42 1159.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1159.92,-45.5001 1169.92,-42 1159.92,-38.5001 1159.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_pool -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer2_pool</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1302,-60C1302,-60 1272,-60 1272,-60 1266,-60 1260,-54 1260,-48 1260,-48 1260,-36 1260,-36 1260,-30 1266,-24 1272,-24 1272,-24 1302,-24 1302,-24 1308,-24 1314,-30 1314,-36 1314,-36 1314,-48 1314,-48 1314,-54 1308,-60 1302,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1287\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_relu&#45;&gt;layer2_pool -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer2_relu&#45;&gt;layer2_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1224.4,-42C1232.39,-42 1241.31,-42 1249.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1249.92,-45.5001 1259.92,-42 1249.92,-38.5001 1249.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_extra_conv -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer2_extra_conv</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1392,-60C1392,-60 1362,-60 1362,-60 1356,-60 1350,-54 1350,-48 1350,-48 1350,-36 1350,-36 1350,-30 1356,-24 1362,-24 1362,-24 1392,-24 1392,-24 1398,-24 1404,-30 1404,-36 1404,-36 1404,-48 1404,-48 1404,-54 1398,-60 1392,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1377\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_pool&#45;&gt;layer2_extra_conv -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer2_pool&#45;&gt;layer2_extra_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1314.4,-42C1322.39,-42 1331.31,-42 1339.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1339.92,-45.5001 1349.92,-42 1339.92,-38.5001 1339.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_extra_bn -->\n",
       "<g id=\"node16\" class=\"node\"><title>layer2_extra_bn</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1482,-60C1482,-60 1452,-60 1452,-60 1446,-60 1440,-54 1440,-48 1440,-48 1440,-36 1440,-36 1440,-30 1446,-24 1452,-24 1452,-24 1482,-24 1482,-24 1488,-24 1494,-30 1494,-36 1494,-36 1494,-48 1494,-48 1494,-54 1488,-60 1482,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1467\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_extra_conv&#45;&gt;layer2_extra_bn -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer2_extra_conv&#45;&gt;layer2_extra_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1404.4,-42C1412.39,-42 1421.31,-42 1429.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1429.92,-45.5001 1439.92,-42 1429.92,-38.5001 1429.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_extra_relu -->\n",
       "<g id=\"node17\" class=\"node\"><title>layer2_extra_relu</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1572,-60C1572,-60 1542,-60 1542,-60 1536,-60 1530,-54 1530,-48 1530,-48 1530,-36 1530,-36 1530,-30 1536,-24 1542,-24 1542,-24 1572,-24 1572,-24 1578,-24 1584,-30 1584,-36 1584,-36 1584,-48 1584,-48 1584,-54 1578,-60 1572,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1557\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_extra_bn&#45;&gt;layer2_extra_relu -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>layer2_extra_bn&#45;&gt;layer2_extra_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1494.4,-42C1502.39,-42 1511.31,-42 1519.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1519.92,-45.5001 1529.92,-42 1519.92,-38.5001 1519.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_conv -->\n",
       "<g id=\"node18\" class=\"node\"><title>layer3_conv</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 512, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1662,-60C1662,-60 1632,-60 1632,-60 1626,-60 1620,-54 1620,-48 1620,-48 1620,-36 1620,-36 1620,-30 1626,-24 1632,-24 1632,-24 1662,-24 1662,-24 1668,-24 1674,-30 1674,-36 1674,-36 1674,-48 1674,-48 1674,-54 1668,-60 1662,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1647\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_extra_relu&#45;&gt;layer3_conv -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>layer2_extra_relu&#45;&gt;layer3_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1584.4,-42C1592.39,-42 1601.31,-42 1609.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1609.92,-45.5001 1619.92,-42 1609.92,-38.5001 1609.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_bn -->\n",
       "<g id=\"node19\" class=\"node\"><title>layer3_bn</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 512, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1752,-60C1752,-60 1722,-60 1722,-60 1716,-60 1710,-54 1710,-48 1710,-48 1710,-36 1710,-36 1710,-30 1716,-24 1722,-24 1722,-24 1752,-24 1752,-24 1758,-24 1764,-30 1764,-36 1764,-36 1764,-48 1764,-48 1764,-54 1758,-60 1752,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1737\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_conv&#45;&gt;layer3_bn -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>layer3_conv&#45;&gt;layer3_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1674.4,-42C1682.39,-42 1691.31,-42 1699.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1699.92,-45.5001 1709.92,-42 1699.92,-38.5001 1699.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_relu -->\n",
       "<g id=\"node20\" class=\"node\"><title>layer3_relu</title>\n",
       "<g id=\"a_node20\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1842,-60C1842,-60 1812,-60 1812,-60 1806,-60 1800,-54 1800,-48 1800,-48 1800,-36 1800,-36 1800,-30 1806,-24 1812,-24 1812,-24 1842,-24 1842,-24 1848,-24 1854,-30 1854,-36 1854,-36 1854,-48 1854,-48 1854,-54 1848,-60 1842,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1827\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_bn&#45;&gt;layer3_relu -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>layer3_bn&#45;&gt;layer3_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1764.4,-42C1772.39,-42 1781.31,-42 1789.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1789.92,-45.5001 1799.92,-42 1789.92,-38.5001 1789.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_pool -->\n",
       "<g id=\"node21\" class=\"node\"><title>layer3_pool</title>\n",
       "<g id=\"a_node21\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1932,-60C1932,-60 1902,-60 1902,-60 1896,-60 1890,-54 1890,-48 1890,-48 1890,-36 1890,-36 1890,-30 1896,-24 1902,-24 1902,-24 1932,-24 1932,-24 1938,-24 1944,-30 1944,-36 1944,-36 1944,-48 1944,-48 1944,-54 1938,-60 1932,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"1917\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_relu&#45;&gt;layer3_pool -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>layer3_relu&#45;&gt;layer3_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1854.4,-42C1862.39,-42 1871.31,-42 1879.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1879.92,-45.5001 1889.92,-42 1879.92,-38.5001 1879.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_extra_conv -->\n",
       "<g id=\"node22\" class=\"node\"><title>layer3_extra_conv</title>\n",
       "<g id=\"a_node22\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 512, &#39;out_channels&#39;: 512, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2022,-60C2022,-60 1992,-60 1992,-60 1986,-60 1980,-54 1980,-48 1980,-48 1980,-36 1980,-36 1980,-30 1986,-24 1992,-24 1992,-24 2022,-24 2022,-24 2028,-24 2034,-30 2034,-36 2034,-36 2034,-48 2034,-48 2034,-54 2028,-60 2022,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"2007\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_pool&#45;&gt;layer3_extra_conv -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>layer3_pool&#45;&gt;layer3_extra_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1944.4,-42C1952.39,-42 1961.31,-42 1969.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1969.92,-45.5001 1979.92,-42 1969.92,-38.5001 1969.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_extra_bn -->\n",
       "<g id=\"node23\" class=\"node\"><title>layer3_extra_bn</title>\n",
       "<g id=\"a_node23\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 512, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2112,-60C2112,-60 2082,-60 2082,-60 2076,-60 2070,-54 2070,-48 2070,-48 2070,-36 2070,-36 2070,-30 2076,-24 2082,-24 2082,-24 2112,-24 2112,-24 2118,-24 2124,-30 2124,-36 2124,-36 2124,-48 2124,-48 2124,-54 2118,-60 2112,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"2097\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_extra_conv&#45;&gt;layer3_extra_bn -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>layer3_extra_conv&#45;&gt;layer3_extra_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2034.4,-42C2042.39,-42 2051.31,-42 2059.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2059.92,-45.5001 2069.92,-42 2059.92,-38.5001 2059.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_extra_relu -->\n",
       "<g id=\"node24\" class=\"node\"><title>layer3_extra_relu</title>\n",
       "<g id=\"a_node24\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2202,-60C2202,-60 2172,-60 2172,-60 2166,-60 2160,-54 2160,-48 2160,-48 2160,-36 2160,-36 2160,-30 2166,-24 2172,-24 2172,-24 2202,-24 2202,-24 2208,-24 2214,-30 2214,-36 2214,-36 2214,-48 2214,-48 2214,-54 2208,-60 2202,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"2187\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_extra_bn&#45;&gt;layer3_extra_relu -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>layer3_extra_bn&#45;&gt;layer3_extra_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2124.4,-42C2132.39,-42 2141.31,-42 2149.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2149.92,-45.5001 2159.92,-42 2149.92,-38.5001 2149.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- pool -->\n",
       "<g id=\"node25\" class=\"node\"><title>pool</title>\n",
       "<g id=\"a_node25\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2292,-60C2292,-60 2262,-60 2262,-60 2256,-60 2250,-54 2250,-48 2250,-48 2250,-36 2250,-36 2250,-30 2256,-24 2262,-24 2262,-24 2292,-24 2292,-24 2298,-24 2304,-30 2304,-36 2304,-36 2304,-48 2304,-48 2304,-54 2298,-60 2292,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"2277\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_extra_relu&#45;&gt;pool -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>layer3_extra_relu&#45;&gt;pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2214.4,-42C2222.39,-42 2231.31,-42 2239.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2239.92,-45.5001 2249.92,-42 2239.92,-38.5001 2239.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- flatten -->\n",
       "<g id=\"node26\" class=\"node\"><title>flatten</title>\n",
       "<g id=\"a_node26\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M2382,-60C2382,-60 2352,-60 2352,-60 2346,-60 2340,-54 2340,-48 2340,-48 2340,-36 2340,-36 2340,-30 2346,-24 2352,-24 2352,-24 2382,-24 2382,-24 2388,-24 2394,-30 2394,-36 2394,-36 2394,-48 2394,-48 2394,-54 2388,-60 2382,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pool&#45;&gt;flatten -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>pool&#45;&gt;flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2304.4,-42C2312.39,-42 2321.31,-42 2329.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2329.92,-45.5001 2339.92,-42 2329.92,-38.5001 2329.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- linear -->\n",
       "<g id=\"node27\" class=\"node\"><title>linear</title>\n",
       "<g id=\"a_node27\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M2472,-60C2472,-60 2442,-60 2442,-60 2436,-60 2430,-54 2430,-48 2430,-48 2430,-36 2430,-36 2430,-30 2436,-24 2442,-24 2442,-24 2472,-24 2472,-24 2478,-24 2484,-30 2484,-36 2484,-36 2484,-48 2484,-48 2484,-54 2478,-60 2472,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"2457\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- flatten&#45;&gt;linear -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>flatten&#45;&gt;linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2394.4,-42C2402.39,-42 2411.31,-42 2419.82,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2419.92,-45.5001 2429.92,-42 2419.92,-38.5001 2419.92,-45.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node28\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node28\"><a xlink:title=\"Mul {&#39;weight&#39;: 0.125}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M2573,-60C2573,-60 2532,-60 2532,-60 2526,-60 2520,-54 2520,-48 2520,-48 2520,-36 2520,-36 2520,-30 2526,-24 2532,-24 2532,-24 2573,-24 2573,-24 2579,-24 2585,-30 2585,-36 2585,-36 2585,-48 2585,-48 2585,-54 2579,-60 2573,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"2552.5\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- linear&#45;&gt;classifier -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2484.04,-42C2492.05,-42 2501.06,-42 2509.83,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2509.95,-45.5001 2519.94,-42 2509.94,-38.5001 2509.95,-45.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node29\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-60C42,-60 12,-60 12,-60 6,-60 0,-54 0,-48 0,-48 0,-36 0,-36 0,-30 6,-24 12,-24 12,-24 42,-24 42,-24 48,-24 54,-30 54,-36 54,-36 54,-48 54,-48 54,-54 48,-60 42,-60\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-42C62.3932,-42 71.3106,-42 79.8241,-42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-45.5001 89.919,-42 79.919,-38.5001 79.919,-45.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7fcd233bda90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.0333       3.4445       1.8571       0.3390       0.2635       1.6723       0.4054       3.7080\n",
      "           2       0.0667       2.6818       1.1796       0.5765       0.1940       0.9478       0.6556       6.5838\n",
      "           3       0.1000       2.7352       0.8991       0.6836       0.1946       1.2571       0.6007       9.5136\n",
      "           4       0.1333       2.6792       0.7638       0.7338       0.1938       0.6789       0.7684      12.3865\n",
      "           5       0.1667       2.6790       0.6880       0.7593       0.1942       0.6666       0.7731      15.2597\n",
      "           6       0.2000       2.6813       0.6320       0.7809       0.1942       0.6575       0.7732      18.1352\n",
      "           7       0.2333       2.7264       0.5933       0.7934       0.1947       0.7307       0.7545      21.0562\n",
      "           8       0.2667       2.6791       0.5655       0.8046       0.1940       0.6260       0.7843      23.9293\n",
      "           9       0.3000       2.6790       0.5375       0.8136       0.1948       0.6303       0.7842      26.8031\n",
      "          10       0.3333       2.6786       0.5313       0.8161       0.1943       0.4720       0.8362      29.6760\n",
      "          11       0.3667       2.7065       0.5086       0.8267       0.1937       0.5271       0.8165      32.5762\n",
      "          12       0.4000       2.6742       0.5110       0.8226       0.1938       0.5090       0.8259      35.4443\n",
      "          13       0.3917       2.6751       0.5047       0.8268       0.1940       0.4656       0.8429      38.3134\n",
      "          14       0.3833       2.6737       0.4788       0.8353       0.1936       0.4469       0.8468      41.1807\n",
      "          15       0.3750       2.7213       0.4698       0.8377       0.1937       0.8202       0.7390      44.0957\n",
      "          16       0.3667       2.6778       0.4583       0.8413       0.1942       0.5731       0.8031      46.9676\n",
      "          17       0.3583       2.6780       0.4502       0.8475       0.1938       0.5557       0.8146      49.8395\n",
      "          18       0.3500       2.6780       0.4430       0.8485       0.1950       0.5101       0.8291      52.7125\n",
      "          19       0.3417       2.6757       0.4339       0.8504       0.2439       0.4668       0.8415      55.6321\n",
      "          20       0.3333       2.6800       0.4265       0.8537       0.1944       0.4982       0.8345      58.5065\n",
      "          21       0.3250       2.6777       0.4098       0.8593       0.1938       0.4719       0.8424      61.3780\n",
      "          22       0.3167       2.6744       0.4100       0.8594       0.1936       0.5227       0.8260      64.2459\n",
      "          23       0.3083       2.6764       0.4023       0.8613       0.2431       0.4889       0.8316      67.1653\n",
      "          24       0.3000       2.6782       0.3921       0.8662       0.1939       0.5267       0.8234      70.0375\n",
      "          25       0.2917       2.6776       0.3899       0.8656       0.1942       0.5875       0.8072      72.9094\n",
      "          26       0.2833       2.6770       0.3866       0.8664       0.1936       0.3986       0.8590      75.7800\n",
      "          27       0.2750       2.6776       0.3762       0.8704       0.2429       0.3791       0.8705      78.7004\n",
      "          28       0.2667       2.6778       0.3681       0.8747       0.1940       0.4105       0.8596      81.5722\n",
      "          29       0.2583       2.6771       0.3639       0.8749       0.1938       0.4449       0.8507      84.4431\n",
      "          30       0.2500       2.6771       0.3527       0.8784       0.1940       0.4001       0.8659      87.3142\n",
      "          31       0.2417       2.6817       0.3458       0.8809       0.2320       0.4669       0.8417      90.2278\n",
      "          32       0.2333       2.6737       0.3428       0.8835       0.1938       0.4075       0.8654      93.0954\n",
      "          33       0.2250       2.6730       0.3371       0.8833       0.1933       0.4057       0.8626      95.9617\n",
      "          34       0.2167       2.6746       0.3316       0.8870       0.1936       0.5017       0.8431      98.8299\n",
      "          35       0.2083       2.6748       0.3207       0.8912       0.2406       0.3497       0.8823     101.7453\n",
      "          36       0.2000       2.6767       0.3159       0.8930       0.1944       0.3279       0.8902     104.6164\n",
      "          37       0.1917       2.6732       0.3085       0.8950       0.1940       0.5001       0.8376     107.4836\n",
      "          38       0.1833       2.6776       0.3037       0.8974       0.1937       0.3622       0.8788     110.3549\n",
      "          39       0.1750       2.6775       0.2953       0.8988       0.1939       0.3427       0.8854     113.2263\n",
      "          40       0.1667       2.7208       0.2874       0.9026       0.1938       0.4025       0.8691     116.1409\n",
      "          41       0.1583       2.6746       0.2821       0.9044       0.1937       0.4346       0.8519     119.0091\n",
      "          42       0.1500       2.6745       0.2751       0.9059       0.1939       0.3542       0.8845     121.8775\n",
      "          43       0.1417       2.6781       0.2658       0.9094       0.1940       0.3034       0.8975     124.7496\n",
      "          44       0.1333       2.7233       0.2548       0.9142       0.1938       0.3474       0.8822     127.6667\n",
      "          45       0.1250       2.6791       0.2471       0.9155       0.1939       0.3214       0.8939     130.5398\n",
      "          46       0.1167       2.6765       0.2390       0.9192       0.1937       0.3004       0.8998     133.4099\n",
      "          47       0.1083       2.6811       0.2339       0.9224       0.1942       0.2805       0.9082     136.2853\n",
      "          48       0.1000       2.7242       0.2207       0.9258       0.1941       0.2664       0.9103     139.2036\n",
      "          49       0.0917       2.6805       0.2090       0.9310       0.1945       0.2615       0.9135     142.0787\n",
      "          50       0.0833       2.6824       0.1995       0.9333       0.1941       0.2336       0.9213     144.9552\n",
      "          51       0.0750       2.6811       0.1883       0.9376       0.1944       0.2621       0.9121     147.8307\n",
      "          52       0.0667       2.7281       0.1750       0.9426       0.1942       0.2246       0.9246     150.7530\n",
      "          53       0.0583       2.6827       0.1650       0.9447       0.1947       0.2265       0.9250     153.6305\n",
      "          54       0.0500       2.6825       0.1509       0.9510       0.1944       0.2011       0.9340     156.5074\n",
      "          55       0.0417       2.6826       0.1392       0.9550       0.1946       0.1908       0.9371     159.3846\n",
      "          56       0.0333       2.7275       0.1234       0.9612       0.1943       0.1834       0.9400     162.3065\n",
      "          57       0.0250       2.6842       0.1107       0.9656       0.1940       0.1771       0.9411     165.1846\n",
      "          58       0.0167       2.6822       0.1005       0.9692       0.1944       0.1718       0.9453     168.0612\n",
      "          59       0.0083       2.6832       0.0883       0.9740       0.1943       0.1627       0.9456     170.9386\n",
      "          60       0.0000       2.7132       0.0801       0.9770       0.1945       0.1592       0.9473     173.8463\n"
     ]
    }
   ],
   "source": [
    "lr_schedule = PiecewiseLinear([0, 12, 60], [0, 0.4, 0])\n",
    "batch_size = 512\n",
    "cutout=12\n",
    "\n",
    "n = net(extra_layers=['layer1', 'layer2', 'layer3'], res_layers=())\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(cutout, cutout)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 4: Architecture](https://www.myrtle.ai/2018/10/26/how_to_train_your_resnet_4/)  - final network Residual:L1+L3, 20 epochs (66s; test acc 93.7%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"66pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 66.01\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.335092 0.335092) rotate(0) translate(4 193)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-193 3219,-193 3219,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-44C94,-44 320,-44 320,-44 326,-44 332,-50 332,-56 332,-56 332,-107 332,-107 332,-113 326,-119 320,-119 320,-119 94,-119 94,-119 88,-119 82,-113 82,-107 82,-107 82,-56 82,-56 82,-50 88,-44 94,-44\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-103.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-8C364,-8 1408,-8 1408,-8 1414,-8 1420,-14 1420,-20 1420,-20 1420,-159 1420,-159 1420,-165 1414,-171 1408,-171 1408,-171 364,-171 364,-171 358,-171 352,-165 352,-159 352,-159 352,-20 352,-20 352,-14 358,-8 364,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"886\" y=\"-155.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer1_residual</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M724,-16C724,-16 1400,-16 1400,-16 1406,-16 1412,-22 1412,-28 1412,-28 1412,-128 1412,-128 1412,-134 1406,-140 1400,-140 1400,-140 724,-140 724,-140 718,-140 712,-134 712,-128 712,-128 712,-28 712,-28 712,-22 718,-16 724,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"1062\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">residual</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer1_residual_res1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M814,-24C814,-24 1040,-24 1040,-24 1046,-24 1052,-30 1052,-36 1052,-36 1052,-87 1052,-87 1052,-93 1046,-99 1040,-99 1040,-99 814,-99 814,-99 808,-99 802,-93 802,-87 802,-87 802,-36 802,-36 802,-30 808,-24 814,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-83.8\" font-family=\"Times,serif\" font-size=\"14.00\">res1</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer1_residual_res2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1084,-24C1084,-24 1310,-24 1310,-24 1316,-24 1322,-30 1322,-36 1322,-36 1322,-87 1322,-87 1322,-93 1316,-99 1310,-99 1310,-99 1084,-99 1084,-99 1078,-99 1072,-93 1072,-87 1072,-87 1072,-36 1072,-36 1072,-30 1078,-24 1084,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-83.8\" font-family=\"Times,serif\" font-size=\"14.00\">res2</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1444,-44C1444,-44 1760,-44 1760,-44 1766,-44 1772,-50 1772,-56 1772,-56 1772,-107 1772,-107 1772,-113 1766,-119 1760,-119 1760,-119 1444,-119 1444,-119 1438,-119 1432,-113 1432,-107 1432,-107 1432,-56 1432,-56 1432,-50 1438,-44 1444,-44\"/>\n",
       "<text text-anchor=\"middle\" x=\"1602\" y=\"-103.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1804,-18C1804,-18 2848,-18 2848,-18 2854,-18 2860,-24 2860,-30 2860,-30 2860,-169 2860,-169 2860,-175 2854,-181 2848,-181 2848,-181 1804,-181 1804,-181 1798,-181 1792,-175 1792,-169 1792,-169 1792,-30 1792,-30 1792,-24 1798,-18 1804,-18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2326\" y=\"-165.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_layer3_residual</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2164,-26C2164,-26 2840,-26 2840,-26 2846,-26 2852,-32 2852,-38 2852,-38 2852,-138 2852,-138 2852,-144 2846,-150 2840,-150 2840,-150 2164,-150 2164,-150 2158,-150 2152,-144 2152,-138 2152,-138 2152,-38 2152,-38 2152,-32 2158,-26 2164,-26\"/>\n",
       "<text text-anchor=\"middle\" x=\"2502\" y=\"-134.8\" font-family=\"Times,serif\" font-size=\"14.00\">residual</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_layer3_residual_res1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2254,-34C2254,-34 2480,-34 2480,-34 2486,-34 2492,-40 2492,-46 2492,-46 2492,-97 2492,-97 2492,-103 2486,-109 2480,-109 2480,-109 2254,-109 2254,-109 2248,-109 2242,-103 2242,-97 2242,-97 2242,-46 2242,-46 2242,-40 2248,-34 2254,-34\"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-93.8\" font-family=\"Times,serif\" font-size=\"14.00\">res1</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_layer3_residual_res2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2524,-34C2524,-34 2750,-34 2750,-34 2756,-34 2762,-40 2762,-46 2762,-46 2762,-97 2762,-97 2762,-103 2756,-109 2750,-109 2750,-109 2524,-109 2524,-109 2518,-109 2512,-103 2512,-97 2512,-97 2512,-46 2512,-46 2512,-40 2518,-34 2524,-34\"/>\n",
       "<text text-anchor=\"middle\" x=\"2637\" y=\"-93.8\" font-family=\"Times,serif\" font-size=\"14.00\">res2</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-88C132,-88 102,-88 102,-88 96,-88 90,-82 90,-76 90,-76 90,-64 90,-64 90,-58 96,-52 102,-52 102,-52 132,-52 132,-52 138,-52 144,-58 144,-64 144,-64 144,-76 144,-76 144,-82 138,-88 132,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn -->\n",
       "<g id=\"node2\" class=\"node\"><title>prep_bn</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-88C222,-88 192,-88 192,-88 186,-88 180,-82 180,-76 180,-76 180,-64 180,-64 180,-58 186,-52 192,-52 192,-52 222,-52 222,-52 228,-52 234,-58 234,-64 234,-64 234,-76 234,-76 234,-82 228,-88 222,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;prep_bn -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;prep_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-70C152.393,-70 161.311,-70 169.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-73.5001 179.919,-70 169.919,-66.5001 169.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- prep_relu -->\n",
       "<g id=\"node3\" class=\"node\"><title>prep_relu</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-88C312,-88 282,-88 282,-88 276,-88 270,-82 270,-76 270,-76 270,-64 270,-64 270,-58 276,-52 282,-52 282,-52 312,-52 312,-52 318,-52 324,-58 324,-64 324,-64 324,-76 324,-76 324,-82 318,-88 312,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn&#45;&gt;prep_relu -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>prep_bn&#45;&gt;prep_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-70C242.393,-70 251.311,-70 259.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-73.5001 269.919,-70 259.919,-66.5001 259.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_conv -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_conv</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-88C402,-88 372,-88 372,-88 366,-88 360,-82 360,-76 360,-76 360,-64 360,-64 360,-58 366,-52 372,-52 372,-52 402,-52 402,-52 408,-52 414,-58 414,-64 414,-64 414,-76 414,-76 414,-82 408,-88 402,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_relu&#45;&gt;layer1_conv -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>prep_relu&#45;&gt;layer1_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-70C332.393,-70 341.311,-70 349.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.919,-73.5001 359.919,-70 349.919,-66.5001 349.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_bn -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_bn</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-88C492,-88 462,-88 462,-88 456,-88 450,-82 450,-76 450,-76 450,-64 450,-64 450,-58 456,-52 462,-52 462,-52 492,-52 492,-52 498,-52 504,-58 504,-64 504,-64 504,-76 504,-76 504,-82 498,-88 492,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_conv&#45;&gt;layer1_bn -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_conv&#45;&gt;layer1_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-70C422.393,-70 431.311,-70 439.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-73.5001 449.919,-70 439.919,-66.5001 439.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_relu -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer1_relu</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-88C582,-88 552,-88 552,-88 546,-88 540,-82 540,-76 540,-76 540,-64 540,-64 540,-58 546,-52 552,-52 552,-52 582,-52 582,-52 588,-52 594,-58 594,-64 594,-64 594,-76 594,-76 594,-82 588,-88 582,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_bn&#45;&gt;layer1_relu -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_bn&#45;&gt;layer1_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-70C512.393,-70 521.311,-70 529.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-73.5001 539.919,-70 529.919,-66.5001 529.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_pool -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer1_pool</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M672,-88C672,-88 642,-88 642,-88 636,-88 630,-82 630,-76 630,-76 630,-64 630,-64 630,-58 636,-52 642,-52 642,-52 672,-52 672,-52 678,-52 684,-58 684,-64 684,-64 684,-76 684,-76 684,-82 678,-88 672,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_relu&#45;&gt;layer1_pool -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer1_relu&#45;&gt;layer1_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-70C602.393,-70 611.311,-70 619.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-73.5001 629.919,-70 619.919,-66.5001 619.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_in -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer1_residual_in</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#ccebc5\" stroke=\"black\" d=\"M762,-88C762,-88 732,-88 732,-88 726,-88 720,-82 720,-76 720,-76 720,-64 720,-64 720,-58 726,-52 732,-52 732,-52 762,-52 762,-52 768,-52 774,-58 774,-64 774,-64 774,-76 774,-76 774,-82 768,-88 762,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"747\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">in</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_pool&#45;&gt;layer1_residual_in -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer1_pool&#45;&gt;layer1_residual_in</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.403,-70C692.393,-70 701.311,-70 709.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"709.919,-73.5001 719.919,-70 709.919,-66.5001 709.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_conv -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer1_residual_res1_conv</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M852,-68C852,-68 822,-68 822,-68 816,-68 810,-62 810,-56 810,-56 810,-44 810,-44 810,-38 816,-32 822,-32 822,-32 852,-32 852,-32 858,-32 864,-38 864,-44 864,-44 864,-56 864,-56 864,-62 858,-68 852,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_in&#45;&gt;layer1_residual_res1_conv -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer1_residual_in&#45;&gt;layer1_residual_res1_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.403,-63.9993C782.481,-62.1634 791.507,-60.1121 800.105,-58.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"800.943,-61.5568 809.919,-55.9275 799.392,-54.7308 800.943,-61.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_add -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer1_residual_add</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#ffed6f\" stroke=\"black\" d=\"M1392,-88C1392,-88 1362,-88 1362,-88 1356,-88 1350,-82 1350,-76 1350,-76 1350,-64 1350,-64 1350,-58 1356,-52 1362,-52 1362,-52 1392,-52 1392,-52 1398,-52 1404,-58 1404,-64 1404,-64 1404,-76 1404,-76 1404,-82 1398,-88 1392,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1377\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_in&#45;&gt;layer1_residual_add -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer1_residual_in&#45;&gt;layer1_residual_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M771.737,-88.1018C780.735,-93.9639 791.381,-99.7798 802,-103 854.782,-119.006 870.844,-108 926,-108 926,-108 926,-108 1198,-108 1253.16,-108 1269.22,-119.006 1322,-103 1329.3,-100.786 1336.61,-97.3454 1343.4,-93.5062\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1345.55,-96.2949 1352.26,-88.1018 1341.9,-90.3176 1345.55,-96.2949\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_bn -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer1_residual_res1_bn</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M942,-68C942,-68 912,-68 912,-68 906,-68 900,-62 900,-56 900,-56 900,-44 900,-44 900,-38 906,-32 912,-32 912,-32 942,-32 942,-32 948,-32 954,-38 954,-44 954,-44 954,-56 954,-56 954,-62 948,-68 942,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_conv&#45;&gt;layer1_residual_res1_bn -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer1_residual_res1_conv&#45;&gt;layer1_residual_res1_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.403,-50C872.393,-50 881.311,-50 889.824,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"889.919,-53.5001 899.919,-50 889.919,-46.5001 889.919,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_relu -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer1_residual_res1_relu</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1032,-68C1032,-68 1002,-68 1002,-68 996,-68 990,-62 990,-56 990,-56 990,-44 990,-44 990,-38 996,-32 1002,-32 1002,-32 1032,-32 1032,-32 1038,-32 1044,-38 1044,-44 1044,-44 1044,-56 1044,-56 1044,-62 1038,-68 1032,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_bn&#45;&gt;layer1_residual_res1_relu -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer1_residual_res1_bn&#45;&gt;layer1_residual_res1_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M954.403,-50C962.393,-50 971.311,-50 979.824,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"979.919,-53.5001 989.919,-50 979.919,-46.5001 979.919,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_conv -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer1_residual_res2_conv</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1122,-68C1122,-68 1092,-68 1092,-68 1086,-68 1080,-62 1080,-56 1080,-56 1080,-44 1080,-44 1080,-38 1086,-32 1092,-32 1092,-32 1122,-32 1122,-32 1128,-32 1134,-38 1134,-44 1134,-44 1134,-56 1134,-56 1134,-62 1128,-68 1122,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_relu&#45;&gt;layer1_residual_res2_conv -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer1_residual_res1_relu&#45;&gt;layer1_residual_res2_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.4,-50C1052.39,-50 1061.31,-50 1069.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1069.92,-53.5001 1079.92,-50 1069.92,-46.5001 1069.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_bn -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer1_residual_res2_bn</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1212,-68C1212,-68 1182,-68 1182,-68 1176,-68 1170,-62 1170,-56 1170,-56 1170,-44 1170,-44 1170,-38 1176,-32 1182,-32 1182,-32 1212,-32 1212,-32 1218,-32 1224,-38 1224,-44 1224,-44 1224,-56 1224,-56 1224,-62 1218,-68 1212,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_conv&#45;&gt;layer1_residual_res2_bn -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer1_residual_res2_conv&#45;&gt;layer1_residual_res2_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1134.4,-50C1142.39,-50 1151.31,-50 1159.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1159.92,-53.5001 1169.92,-50 1159.92,-46.5001 1159.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_relu -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer1_residual_res2_relu</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1302,-68C1302,-68 1272,-68 1272,-68 1266,-68 1260,-62 1260,-56 1260,-56 1260,-44 1260,-44 1260,-38 1266,-32 1272,-32 1272,-32 1302,-32 1302,-32 1308,-32 1314,-38 1314,-44 1314,-44 1314,-56 1314,-56 1314,-62 1308,-68 1302,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"1287\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_bn&#45;&gt;layer1_residual_res2_relu -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer1_residual_res2_bn&#45;&gt;layer1_residual_res2_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1224.4,-50C1232.39,-50 1241.31,-50 1249.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1249.92,-53.5001 1259.92,-50 1249.92,-46.5001 1249.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_relu&#45;&gt;layer1_residual_add -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer1_residual_res2_relu&#45;&gt;layer1_residual_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1314.4,-56.0007C1322.48,-57.8366 1331.51,-59.8879 1340.1,-61.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1339.39,-65.2692 1349.92,-64.0725 1340.94,-58.4432 1339.39,-65.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_conv -->\n",
       "<g id=\"node16\" class=\"node\"><title>layer2_conv</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1482,-88C1482,-88 1452,-88 1452,-88 1446,-88 1440,-82 1440,-76 1440,-76 1440,-64 1440,-64 1440,-58 1446,-52 1452,-52 1452,-52 1482,-52 1482,-52 1488,-52 1494,-58 1494,-64 1494,-64 1494,-76 1494,-76 1494,-82 1488,-88 1482,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1467\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_add&#45;&gt;layer2_conv -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>layer1_residual_add&#45;&gt;layer2_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1404.4,-70C1412.39,-70 1421.31,-70 1429.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1429.92,-73.5001 1439.92,-70 1429.92,-66.5001 1429.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_bn -->\n",
       "<g id=\"node17\" class=\"node\"><title>layer2_bn</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1572,-88C1572,-88 1542,-88 1542,-88 1536,-88 1530,-82 1530,-76 1530,-76 1530,-64 1530,-64 1530,-58 1536,-52 1542,-52 1542,-52 1572,-52 1572,-52 1578,-52 1584,-58 1584,-64 1584,-64 1584,-76 1584,-76 1584,-82 1578,-88 1572,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1557\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_conv&#45;&gt;layer2_bn -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>layer2_conv&#45;&gt;layer2_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1494.4,-70C1502.39,-70 1511.31,-70 1519.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1519.92,-73.5001 1529.92,-70 1519.92,-66.5001 1519.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_relu -->\n",
       "<g id=\"node18\" class=\"node\"><title>layer2_relu</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1662,-88C1662,-88 1632,-88 1632,-88 1626,-88 1620,-82 1620,-76 1620,-76 1620,-64 1620,-64 1620,-58 1626,-52 1632,-52 1632,-52 1662,-52 1662,-52 1668,-52 1674,-58 1674,-64 1674,-64 1674,-76 1674,-76 1674,-82 1668,-88 1662,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1647\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_bn&#45;&gt;layer2_relu -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>layer2_bn&#45;&gt;layer2_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1584.4,-70C1592.39,-70 1601.31,-70 1609.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1609.92,-73.5001 1619.92,-70 1609.92,-66.5001 1609.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_pool -->\n",
       "<g id=\"node19\" class=\"node\"><title>layer2_pool</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1752,-88C1752,-88 1722,-88 1722,-88 1716,-88 1710,-82 1710,-76 1710,-76 1710,-64 1710,-64 1710,-58 1716,-52 1722,-52 1722,-52 1752,-52 1752,-52 1758,-52 1764,-58 1764,-64 1764,-64 1764,-76 1764,-76 1764,-82 1758,-88 1752,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1737\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_relu&#45;&gt;layer2_pool -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>layer2_relu&#45;&gt;layer2_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1674.4,-70C1682.39,-70 1691.31,-70 1699.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1699.92,-73.5001 1709.92,-70 1699.92,-66.5001 1699.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_conv -->\n",
       "<g id=\"node20\" class=\"node\"><title>layer3_conv</title>\n",
       "<g id=\"a_node20\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 512, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1842,-88C1842,-88 1812,-88 1812,-88 1806,-88 1800,-82 1800,-76 1800,-76 1800,-64 1800,-64 1800,-58 1806,-52 1812,-52 1812,-52 1842,-52 1842,-52 1848,-52 1854,-58 1854,-64 1854,-64 1854,-76 1854,-76 1854,-82 1848,-88 1842,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1827\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_pool&#45;&gt;layer3_conv -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>layer2_pool&#45;&gt;layer3_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1764.4,-70C1772.39,-70 1781.31,-70 1789.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1789.92,-73.5001 1799.92,-70 1789.92,-66.5001 1789.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_bn -->\n",
       "<g id=\"node21\" class=\"node\"><title>layer3_bn</title>\n",
       "<g id=\"a_node21\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 512, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1932,-88C1932,-88 1902,-88 1902,-88 1896,-88 1890,-82 1890,-76 1890,-76 1890,-64 1890,-64 1890,-58 1896,-52 1902,-52 1902,-52 1932,-52 1932,-52 1938,-52 1944,-58 1944,-64 1944,-64 1944,-76 1944,-76 1944,-82 1938,-88 1932,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1917\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_conv&#45;&gt;layer3_bn -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>layer3_conv&#45;&gt;layer3_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1854.4,-70C1862.39,-70 1871.31,-70 1879.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1879.92,-73.5001 1889.92,-70 1879.92,-66.5001 1879.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_relu -->\n",
       "<g id=\"node22\" class=\"node\"><title>layer3_relu</title>\n",
       "<g id=\"a_node22\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2022,-88C2022,-88 1992,-88 1992,-88 1986,-88 1980,-82 1980,-76 1980,-76 1980,-64 1980,-64 1980,-58 1986,-52 1992,-52 1992,-52 2022,-52 2022,-52 2028,-52 2034,-58 2034,-64 2034,-64 2034,-76 2034,-76 2034,-82 2028,-88 2022,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"2007\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_bn&#45;&gt;layer3_relu -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>layer3_bn&#45;&gt;layer3_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1944.4,-70C1952.39,-70 1961.31,-70 1969.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1969.92,-73.5001 1979.92,-70 1969.92,-66.5001 1969.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_pool -->\n",
       "<g id=\"node23\" class=\"node\"><title>layer3_pool</title>\n",
       "<g id=\"a_node23\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2112,-88C2112,-88 2082,-88 2082,-88 2076,-88 2070,-82 2070,-76 2070,-76 2070,-64 2070,-64 2070,-58 2076,-52 2082,-52 2082,-52 2112,-52 2112,-52 2118,-52 2124,-58 2124,-64 2124,-64 2124,-76 2124,-76 2124,-82 2118,-88 2112,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"2097\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_relu&#45;&gt;layer3_pool -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>layer3_relu&#45;&gt;layer3_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2034.4,-70C2042.39,-70 2051.31,-70 2059.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2059.92,-73.5001 2069.92,-70 2059.92,-66.5001 2059.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_in -->\n",
       "<g id=\"node24\" class=\"node\"><title>layer3_residual_in</title>\n",
       "<g id=\"a_node24\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#ccebc5\" stroke=\"black\" d=\"M2202,-88C2202,-88 2172,-88 2172,-88 2166,-88 2160,-82 2160,-76 2160,-76 2160,-64 2160,-64 2160,-58 2166,-52 2172,-52 2172,-52 2202,-52 2202,-52 2208,-52 2214,-58 2214,-64 2214,-64 2214,-76 2214,-76 2214,-82 2208,-88 2202,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"2187\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">in</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_pool&#45;&gt;layer3_residual_in -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>layer3_pool&#45;&gt;layer3_residual_in</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2124.4,-70C2132.39,-70 2141.31,-70 2149.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2149.92,-73.5001 2159.92,-70 2149.92,-66.5001 2149.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_conv -->\n",
       "<g id=\"node25\" class=\"node\"><title>layer3_residual_res1_conv</title>\n",
       "<g id=\"a_node25\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 512, &#39;out_channels&#39;: 512, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2292,-78C2292,-78 2262,-78 2262,-78 2256,-78 2250,-72 2250,-66 2250,-66 2250,-54 2250,-54 2250,-48 2256,-42 2262,-42 2262,-42 2292,-42 2292,-42 2298,-42 2304,-48 2304,-54 2304,-54 2304,-66 2304,-66 2304,-72 2298,-78 2292,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2277\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_in&#45;&gt;layer3_residual_res1_conv -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>layer3_residual_in&#45;&gt;layer3_residual_res1_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2214.4,-66.9997C2222.39,-66.0917 2231.31,-65.0783 2239.82,-64.1109\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2240.38,-67.5706 2249.92,-62.9638 2239.59,-60.6153 2240.38,-67.5706\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_add -->\n",
       "<g id=\"node31\" class=\"node\"><title>layer3_residual_add</title>\n",
       "<g id=\"a_node31\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#ffed6f\" stroke=\"black\" d=\"M2832,-98C2832,-98 2802,-98 2802,-98 2796,-98 2790,-92 2790,-86 2790,-86 2790,-74 2790,-74 2790,-68 2796,-62 2802,-62 2802,-62 2832,-62 2832,-62 2838,-62 2844,-68 2844,-74 2844,-74 2844,-86 2844,-86 2844,-92 2838,-98 2832,-98\"/>\n",
       "<text text-anchor=\"middle\" x=\"2817\" y=\"-76.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_in&#45;&gt;layer3_residual_add -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>layer3_residual_in&#45;&gt;layer3_residual_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2205.16,-88.3721C2214.97,-97.5647 2228.07,-107.815 2242,-113 2293.69,-132.244 2310.84,-118 2366,-118 2366,-118 2366,-118 2638,-118 2693.16,-118 2709.22,-129.006 2762,-113 2769.3,-110.786 2776.61,-107.345 2783.4,-103.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2785.55,-106.295 2792.26,-98.1018 2781.9,-100.318 2785.55,-106.295\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_bn -->\n",
       "<g id=\"node26\" class=\"node\"><title>layer3_residual_res1_bn</title>\n",
       "<g id=\"a_node26\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 512, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2382,-78C2382,-78 2352,-78 2352,-78 2346,-78 2340,-72 2340,-66 2340,-66 2340,-54 2340,-54 2340,-48 2346,-42 2352,-42 2352,-42 2382,-42 2382,-42 2388,-42 2394,-48 2394,-54 2394,-54 2394,-66 2394,-66 2394,-72 2388,-78 2382,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_conv&#45;&gt;layer3_residual_res1_bn -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>layer3_residual_res1_conv&#45;&gt;layer3_residual_res1_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2304.4,-60C2312.39,-60 2321.31,-60 2329.82,-60\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2329.92,-63.5001 2339.92,-60 2329.92,-56.5001 2329.92,-63.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_relu -->\n",
       "<g id=\"node27\" class=\"node\"><title>layer3_residual_res1_relu</title>\n",
       "<g id=\"a_node27\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2472,-78C2472,-78 2442,-78 2442,-78 2436,-78 2430,-72 2430,-66 2430,-66 2430,-54 2430,-54 2430,-48 2436,-42 2442,-42 2442,-42 2472,-42 2472,-42 2478,-42 2484,-48 2484,-54 2484,-54 2484,-66 2484,-66 2484,-72 2478,-78 2472,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2457\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_bn&#45;&gt;layer3_residual_res1_relu -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>layer3_residual_res1_bn&#45;&gt;layer3_residual_res1_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2394.4,-60C2402.39,-60 2411.31,-60 2419.82,-60\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2419.92,-63.5001 2429.92,-60 2419.92,-56.5001 2419.92,-63.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_conv -->\n",
       "<g id=\"node28\" class=\"node\"><title>layer3_residual_res2_conv</title>\n",
       "<g id=\"a_node28\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 512, &#39;out_channels&#39;: 512, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2562,-78C2562,-78 2532,-78 2532,-78 2526,-78 2520,-72 2520,-66 2520,-66 2520,-54 2520,-54 2520,-48 2526,-42 2532,-42 2532,-42 2562,-42 2562,-42 2568,-42 2574,-48 2574,-54 2574,-54 2574,-66 2574,-66 2574,-72 2568,-78 2562,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2547\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_relu&#45;&gt;layer3_residual_res2_conv -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>layer3_residual_res1_relu&#45;&gt;layer3_residual_res2_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2484.4,-60C2492.39,-60 2501.31,-60 2509.82,-60\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2509.92,-63.5001 2519.92,-60 2509.92,-56.5001 2509.92,-63.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_bn -->\n",
       "<g id=\"node29\" class=\"node\"><title>layer3_residual_res2_bn</title>\n",
       "<g id=\"a_node29\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 512, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2652,-78C2652,-78 2622,-78 2622,-78 2616,-78 2610,-72 2610,-66 2610,-66 2610,-54 2610,-54 2610,-48 2616,-42 2622,-42 2622,-42 2652,-42 2652,-42 2658,-42 2664,-48 2664,-54 2664,-54 2664,-66 2664,-66 2664,-72 2658,-78 2652,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2637\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_conv&#45;&gt;layer3_residual_res2_bn -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>layer3_residual_res2_conv&#45;&gt;layer3_residual_res2_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2574.4,-60C2582.39,-60 2591.31,-60 2599.82,-60\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2599.92,-63.5001 2609.92,-60 2599.92,-56.5001 2599.92,-63.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_relu -->\n",
       "<g id=\"node30\" class=\"node\"><title>layer3_residual_res2_relu</title>\n",
       "<g id=\"a_node30\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2742,-78C2742,-78 2712,-78 2712,-78 2706,-78 2700,-72 2700,-66 2700,-66 2700,-54 2700,-54 2700,-48 2706,-42 2712,-42 2712,-42 2742,-42 2742,-42 2748,-42 2754,-48 2754,-54 2754,-54 2754,-66 2754,-66 2754,-72 2748,-78 2742,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2727\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_bn&#45;&gt;layer3_residual_res2_relu -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>layer3_residual_res2_bn&#45;&gt;layer3_residual_res2_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2664.4,-60C2672.39,-60 2681.31,-60 2689.82,-60\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2689.92,-63.5001 2699.92,-60 2689.92,-56.5001 2689.92,-63.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_relu&#45;&gt;layer3_residual_add -->\n",
       "<g id=\"edge33\" class=\"edge\"><title>layer3_residual_res2_relu&#45;&gt;layer3_residual_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2754.4,-66.0007C2762.48,-67.8366 2771.51,-69.8879 2780.1,-71.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2779.39,-75.2692 2789.92,-74.0725 2780.94,-68.4432 2779.39,-75.2692\"/>\n",
       "</g>\n",
       "<!-- pool -->\n",
       "<g id=\"node32\" class=\"node\"><title>pool</title>\n",
       "<g id=\"a_node32\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2922,-98C2922,-98 2892,-98 2892,-98 2886,-98 2880,-92 2880,-86 2880,-86 2880,-74 2880,-74 2880,-68 2886,-62 2892,-62 2892,-62 2922,-62 2922,-62 2928,-62 2934,-68 2934,-74 2934,-74 2934,-86 2934,-86 2934,-92 2928,-98 2922,-98\"/>\n",
       "<text text-anchor=\"middle\" x=\"2907\" y=\"-76.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_add&#45;&gt;pool -->\n",
       "<g id=\"edge34\" class=\"edge\"><title>layer3_residual_add&#45;&gt;pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2844.4,-80C2852.39,-80 2861.31,-80 2869.82,-80\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2869.92,-83.5001 2879.92,-80 2869.92,-76.5001 2869.92,-83.5001\"/>\n",
       "</g>\n",
       "<!-- flatten -->\n",
       "<g id=\"node33\" class=\"node\"><title>flatten</title>\n",
       "<g id=\"a_node33\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M3012,-98C3012,-98 2982,-98 2982,-98 2976,-98 2970,-92 2970,-86 2970,-86 2970,-74 2970,-74 2970,-68 2976,-62 2982,-62 2982,-62 3012,-62 3012,-62 3018,-62 3024,-68 3024,-74 3024,-74 3024,-86 3024,-86 3024,-92 3018,-98 3012,-98\"/>\n",
       "<text text-anchor=\"middle\" x=\"2997\" y=\"-76.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pool&#45;&gt;flatten -->\n",
       "<g id=\"edge35\" class=\"edge\"><title>pool&#45;&gt;flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2934.4,-80C2942.39,-80 2951.31,-80 2959.82,-80\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2959.92,-83.5001 2969.92,-80 2959.92,-76.5001 2959.92,-83.5001\"/>\n",
       "</g>\n",
       "<!-- linear -->\n",
       "<g id=\"node34\" class=\"node\"><title>linear</title>\n",
       "<g id=\"a_node34\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M3102,-98C3102,-98 3072,-98 3072,-98 3066,-98 3060,-92 3060,-86 3060,-86 3060,-74 3060,-74 3060,-68 3066,-62 3072,-62 3072,-62 3102,-62 3102,-62 3108,-62 3114,-68 3114,-74 3114,-74 3114,-86 3114,-86 3114,-92 3108,-98 3102,-98\"/>\n",
       "<text text-anchor=\"middle\" x=\"3087\" y=\"-76.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- flatten&#45;&gt;linear -->\n",
       "<g id=\"edge36\" class=\"edge\"><title>flatten&#45;&gt;linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3024.4,-80C3032.39,-80 3041.31,-80 3049.82,-80\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3049.92,-83.5001 3059.92,-80 3049.92,-76.5001 3049.92,-83.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node35\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node35\"><a xlink:title=\"Mul {&#39;weight&#39;: 0.125}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M3203,-98C3203,-98 3162,-98 3162,-98 3156,-98 3150,-92 3150,-86 3150,-86 3150,-74 3150,-74 3150,-68 3156,-62 3162,-62 3162,-62 3203,-62 3203,-62 3209,-62 3215,-68 3215,-74 3215,-74 3215,-86 3215,-86 3215,-92 3209,-98 3203,-98\"/>\n",
       "<text text-anchor=\"middle\" x=\"3182.5\" y=\"-76.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- linear&#45;&gt;classifier -->\n",
       "<g id=\"edge37\" class=\"edge\"><title>linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3114.04,-80C3122.05,-80 3131.06,-80 3139.83,-80\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3139.95,-83.5001 3149.94,-80 3139.94,-76.5001 3139.95,-83.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node36\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-88C42,-88 12,-88 12,-88 6,-88 0,-82 0,-76 0,-76 0,-64 0,-64 0,-58 6,-52 12,-52 12,-52 42,-52 42,-52 48,-52 54,-58 54,-64 54,-64 54,-76 54,-76 54,-82 48,-88 42,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-70C62.3932,-70 71.3106,-70 79.8241,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-73.5001 89.919,-70 79.919,-66.5001 79.919,-73.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7fcd23cb7b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.1000       2.9605       1.5841       0.4281       0.2195       1.1686       0.5865       3.1801\n",
      "           2       0.2000       2.9629       0.9454       0.6631       0.2759       0.8585       0.7054       6.4188\n",
      "           3       0.3000       2.9642       0.7213       0.7491       0.2190       0.7146       0.7409       9.6020\n",
      "           4       0.4000       2.9646       0.6209       0.7841       0.2186       0.8419       0.7226      12.7852\n",
      "           5       0.3750       2.9637       0.5321       0.8178       0.2186       0.5625       0.8072      15.9675\n",
      "           6       0.3500       2.9598       0.4671       0.8410       0.2688       0.6967       0.7550      19.1960\n",
      "           7       0.3250       2.9616       0.4235       0.8552       0.2188       0.7235       0.7547      22.3765\n",
      "           8       0.3000       2.9597       0.3848       0.8694       0.2182       0.4751       0.8363      25.5543\n",
      "           9       0.2750       2.9586       0.3670       0.8753       0.2185       0.4868       0.8371      28.7314\n",
      "          10       0.2500       2.9589       0.3459       0.8799       0.2721       0.4221       0.8568      31.9623\n",
      "          11       0.2250       2.9584       0.3207       0.8903       0.2183       0.4899       0.8365      35.1390\n",
      "          12       0.2000       2.9577       0.2981       0.8996       0.2185       0.3828       0.8725      38.3152\n",
      "          13       0.1750       2.9576       0.2763       0.9070       0.2182       0.4204       0.8564      41.4910\n",
      "          14       0.1500       2.9540       0.2495       0.9151       0.2657       0.3192       0.8944      44.7107\n",
      "          15       0.1250       2.9573       0.2311       0.9227       0.2186       0.3432       0.8836      47.8866\n",
      "          16       0.1000       2.9551       0.2014       0.9324       0.2189       0.2937       0.9042      51.0607\n",
      "          17       0.0750       2.9613       0.1746       0.9422       0.2181       0.2567       0.9127      54.2400\n",
      "          18       0.0500       2.9610       0.1431       0.9547       0.2182       0.2387       0.9224      57.4192\n",
      "          19       0.0250       3.0093       0.1195       0.9621       0.2178       0.2172       0.9250      60.6462\n",
      "          20       0.0000       2.9576       0.0936       0.9727       0.2183       0.1841       0.9383      63.8221\n"
     ]
    }
   ],
   "source": [
    "lr_schedule = PiecewiseLinear([0, 4, 20], [0, 0.4, 0])\n",
    "batch_size = 512\n",
    "\n",
    "n = net()\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(8,8)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Post 4: Architecture](https://www.myrtle.ai/2018/10/26/how_to_train_your_resnet_4/)  - final network, 24 epochs (79s; test acc 94.1%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1080pt\" height=\"66pt\"\n",
       " viewBox=\"0.00 0.00 1080.00 66.01\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.335092 0.335092) rotate(0) translate(4 193)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-193 3219,-193 3219,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_prep</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M94,-44C94,-44 320,-44 320,-44 326,-44 332,-50 332,-56 332,-56 332,-107 332,-107 332,-113 326,-119 320,-119 320,-119 94,-119 94,-119 88,-119 82,-113 82,-107 82,-107 82,-56 82,-56 82,-50 88,-44 94,-44\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-103.8\" font-family=\"Times,serif\" font-size=\"14.00\">prep</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_layer1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M364,-8C364,-8 1408,-8 1408,-8 1414,-8 1420,-14 1420,-20 1420,-20 1420,-159 1420,-159 1420,-165 1414,-171 1408,-171 1408,-171 364,-171 364,-171 358,-171 352,-165 352,-159 352,-159 352,-20 352,-20 352,-14 358,-8 364,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"886\" y=\"-155.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer1</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_layer1_residual</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M724,-16C724,-16 1400,-16 1400,-16 1406,-16 1412,-22 1412,-28 1412,-28 1412,-128 1412,-128 1412,-134 1406,-140 1400,-140 1400,-140 724,-140 724,-140 718,-140 712,-134 712,-128 712,-128 712,-28 712,-28 712,-22 718,-16 724,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"1062\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">residual</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_layer1_residual_res1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M814,-24C814,-24 1040,-24 1040,-24 1046,-24 1052,-30 1052,-36 1052,-36 1052,-87 1052,-87 1052,-93 1046,-99 1040,-99 1040,-99 814,-99 814,-99 808,-99 802,-93 802,-87 802,-87 802,-36 802,-36 802,-30 808,-24 814,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-83.8\" font-family=\"Times,serif\" font-size=\"14.00\">res1</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_layer1_residual_res2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1084,-24C1084,-24 1310,-24 1310,-24 1316,-24 1322,-30 1322,-36 1322,-36 1322,-87 1322,-87 1322,-93 1316,-99 1310,-99 1310,-99 1084,-99 1084,-99 1078,-99 1072,-93 1072,-87 1072,-87 1072,-36 1072,-36 1072,-30 1078,-24 1084,-24\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-83.8\" font-family=\"Times,serif\" font-size=\"14.00\">res2</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_layer2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1444,-44C1444,-44 1760,-44 1760,-44 1766,-44 1772,-50 1772,-56 1772,-56 1772,-107 1772,-107 1772,-113 1766,-119 1760,-119 1760,-119 1444,-119 1444,-119 1438,-119 1432,-113 1432,-107 1432,-107 1432,-56 1432,-56 1432,-50 1438,-44 1444,-44\"/>\n",
       "<text text-anchor=\"middle\" x=\"1602\" y=\"-103.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer2</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_layer3</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M1804,-18C1804,-18 2848,-18 2848,-18 2854,-18 2860,-24 2860,-30 2860,-30 2860,-169 2860,-169 2860,-175 2854,-181 2848,-181 2848,-181 1804,-181 1804,-181 1798,-181 1792,-175 1792,-169 1792,-169 1792,-30 1792,-30 1792,-24 1798,-18 1804,-18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2326\" y=\"-165.8\" font-family=\"Times,serif\" font-size=\"14.00\">layer3</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_layer3_residual</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2164,-26C2164,-26 2840,-26 2840,-26 2846,-26 2852,-32 2852,-38 2852,-38 2852,-138 2852,-138 2852,-144 2846,-150 2840,-150 2840,-150 2164,-150 2164,-150 2158,-150 2152,-144 2152,-138 2152,-138 2152,-38 2152,-38 2152,-32 2158,-26 2164,-26\"/>\n",
       "<text text-anchor=\"middle\" x=\"2502\" y=\"-134.8\" font-family=\"Times,serif\" font-size=\"14.00\">residual</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_layer3_residual_res1</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2254,-34C2254,-34 2480,-34 2480,-34 2486,-34 2492,-40 2492,-46 2492,-46 2492,-97 2492,-97 2492,-103 2486,-109 2480,-109 2480,-109 2254,-109 2254,-109 2248,-109 2242,-103 2242,-97 2242,-97 2242,-46 2242,-46 2242,-40 2248,-34 2254,-34\"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-93.8\" font-family=\"Times,serif\" font-size=\"14.00\">res1</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_layer3_residual_res2</title>\n",
       "<path fill=\"#777777\" fill-opacity=\"0.266667\" stroke=\"black\" d=\"M2524,-34C2524,-34 2750,-34 2750,-34 2756,-34 2762,-40 2762,-46 2762,-46 2762,-97 2762,-97 2762,-103 2756,-109 2750,-109 2750,-109 2524,-109 2524,-109 2518,-109 2512,-103 2512,-97 2512,-97 2512,-46 2512,-46 2512,-40 2518,-34 2524,-34\"/>\n",
       "<text text-anchor=\"middle\" x=\"2637\" y=\"-93.8\" font-family=\"Times,serif\" font-size=\"14.00\">res2</text>\n",
       "</g>\n",
       "<!-- prep_conv -->\n",
       "<g id=\"node1\" class=\"node\"><title>prep_conv</title>\n",
       "<g id=\"a_node1\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 3, &#39;out_channels&#39;: 64, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M132,-88C132,-88 102,-88 102,-88 96,-88 90,-82 90,-76 90,-76 90,-64 90,-64 90,-58 96,-52 102,-52 102,-52 132,-52 132,-52 138,-52 144,-58 144,-64 144,-64 144,-76 144,-76 144,-82 138,-88 132,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn -->\n",
       "<g id=\"node2\" class=\"node\"><title>prep_bn</title>\n",
       "<g id=\"a_node2\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 64, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M222,-88C222,-88 192,-88 192,-88 186,-88 180,-82 180,-76 180,-76 180,-64 180,-64 180,-58 186,-52 192,-52 192,-52 222,-52 222,-52 228,-52 234,-58 234,-64 234,-64 234,-76 234,-76 234,-82 228,-88 222,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_conv&#45;&gt;prep_bn -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>prep_conv&#45;&gt;prep_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M144.403,-70C152.393,-70 161.311,-70 169.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.919,-73.5001 179.919,-70 169.919,-66.5001 169.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- prep_relu -->\n",
       "<g id=\"node3\" class=\"node\"><title>prep_relu</title>\n",
       "<g id=\"a_node3\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M312,-88C312,-88 282,-88 282,-88 276,-88 270,-82 270,-76 270,-76 270,-64 270,-64 270,-58 276,-52 282,-52 282,-52 312,-52 312,-52 318,-52 324,-58 324,-64 324,-64 324,-76 324,-76 324,-82 318,-88 312,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_bn&#45;&gt;prep_relu -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>prep_bn&#45;&gt;prep_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.403,-70C242.393,-70 251.311,-70 259.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"259.919,-73.5001 269.919,-70 259.919,-66.5001 259.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_conv -->\n",
       "<g id=\"node4\" class=\"node\"><title>layer1_conv</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 64, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M402,-88C402,-88 372,-88 372,-88 366,-88 360,-82 360,-76 360,-76 360,-64 360,-64 360,-58 366,-52 372,-52 372,-52 402,-52 402,-52 408,-52 414,-58 414,-64 414,-64 414,-76 414,-76 414,-82 408,-88 402,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"387\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- prep_relu&#45;&gt;layer1_conv -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>prep_relu&#45;&gt;layer1_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.403,-70C332.393,-70 341.311,-70 349.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.919,-73.5001 359.919,-70 349.919,-66.5001 349.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_bn -->\n",
       "<g id=\"node5\" class=\"node\"><title>layer1_bn</title>\n",
       "<g id=\"a_node5\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M492,-88C492,-88 462,-88 462,-88 456,-88 450,-82 450,-76 450,-76 450,-64 450,-64 450,-58 456,-52 462,-52 462,-52 492,-52 492,-52 498,-52 504,-58 504,-64 504,-64 504,-76 504,-76 504,-82 498,-88 492,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"477\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_conv&#45;&gt;layer1_bn -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>layer1_conv&#45;&gt;layer1_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414.403,-70C422.393,-70 431.311,-70 439.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"439.919,-73.5001 449.919,-70 439.919,-66.5001 439.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_relu -->\n",
       "<g id=\"node6\" class=\"node\"><title>layer1_relu</title>\n",
       "<g id=\"a_node6\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M582,-88C582,-88 552,-88 552,-88 546,-88 540,-82 540,-76 540,-76 540,-64 540,-64 540,-58 546,-52 552,-52 552,-52 582,-52 582,-52 588,-52 594,-58 594,-64 594,-64 594,-76 594,-76 594,-82 588,-88 582,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_bn&#45;&gt;layer1_relu -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>layer1_bn&#45;&gt;layer1_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.403,-70C512.393,-70 521.311,-70 529.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"529.919,-73.5001 539.919,-70 529.919,-66.5001 529.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_pool -->\n",
       "<g id=\"node7\" class=\"node\"><title>layer1_pool</title>\n",
       "<g id=\"a_node7\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M672,-88C672,-88 642,-88 642,-88 636,-88 630,-82 630,-76 630,-76 630,-64 630,-64 630,-58 636,-52 642,-52 642,-52 672,-52 672,-52 678,-52 684,-58 684,-64 684,-64 684,-76 684,-76 684,-82 678,-88 672,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"657\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_relu&#45;&gt;layer1_pool -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>layer1_relu&#45;&gt;layer1_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.403,-70C602.393,-70 611.311,-70 619.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"619.919,-73.5001 629.919,-70 619.919,-66.5001 619.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_in -->\n",
       "<g id=\"node8\" class=\"node\"><title>layer1_residual_in</title>\n",
       "<g id=\"a_node8\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#ccebc5\" stroke=\"black\" d=\"M762,-88C762,-88 732,-88 732,-88 726,-88 720,-82 720,-76 720,-76 720,-64 720,-64 720,-58 726,-52 732,-52 732,-52 762,-52 762,-52 768,-52 774,-58 774,-64 774,-64 774,-76 774,-76 774,-82 768,-88 762,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"747\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">in</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_pool&#45;&gt;layer1_residual_in -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>layer1_pool&#45;&gt;layer1_residual_in</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.403,-70C692.393,-70 701.311,-70 709.824,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"709.919,-73.5001 719.919,-70 709.919,-66.5001 709.919,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_conv -->\n",
       "<g id=\"node9\" class=\"node\"><title>layer1_residual_res1_conv</title>\n",
       "<g id=\"a_node9\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M852,-68C852,-68 822,-68 822,-68 816,-68 810,-62 810,-56 810,-56 810,-44 810,-44 810,-38 816,-32 822,-32 822,-32 852,-32 852,-32 858,-32 864,-38 864,-44 864,-44 864,-56 864,-56 864,-62 858,-68 852,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_in&#45;&gt;layer1_residual_res1_conv -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>layer1_residual_in&#45;&gt;layer1_residual_res1_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.403,-63.9993C782.481,-62.1634 791.507,-60.1121 800.105,-58.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"800.943,-61.5568 809.919,-55.9275 799.392,-54.7308 800.943,-61.5568\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_add -->\n",
       "<g id=\"node15\" class=\"node\"><title>layer1_residual_add</title>\n",
       "<g id=\"a_node15\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#ffed6f\" stroke=\"black\" d=\"M1392,-88C1392,-88 1362,-88 1362,-88 1356,-88 1350,-82 1350,-76 1350,-76 1350,-64 1350,-64 1350,-58 1356,-52 1362,-52 1362,-52 1392,-52 1392,-52 1398,-52 1404,-58 1404,-64 1404,-64 1404,-76 1404,-76 1404,-82 1398,-88 1392,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1377\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_in&#45;&gt;layer1_residual_add -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>layer1_residual_in&#45;&gt;layer1_residual_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M771.737,-88.1018C780.735,-93.9639 791.381,-99.7798 802,-103 854.782,-119.006 870.844,-108 926,-108 926,-108 926,-108 1198,-108 1253.16,-108 1269.22,-119.006 1322,-103 1329.3,-100.786 1336.61,-97.3454 1343.4,-93.5062\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1345.55,-96.2949 1352.26,-88.1018 1341.9,-90.3176 1345.55,-96.2949\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_bn -->\n",
       "<g id=\"node10\" class=\"node\"><title>layer1_residual_res1_bn</title>\n",
       "<g id=\"a_node10\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M942,-68C942,-68 912,-68 912,-68 906,-68 900,-62 900,-56 900,-56 900,-44 900,-44 900,-38 906,-32 912,-32 912,-32 942,-32 942,-32 948,-32 954,-38 954,-44 954,-44 954,-56 954,-56 954,-62 948,-68 942,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"927\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_conv&#45;&gt;layer1_residual_res1_bn -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>layer1_residual_res1_conv&#45;&gt;layer1_residual_res1_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.403,-50C872.393,-50 881.311,-50 889.824,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"889.919,-53.5001 899.919,-50 889.919,-46.5001 889.919,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_relu -->\n",
       "<g id=\"node11\" class=\"node\"><title>layer1_residual_res1_relu</title>\n",
       "<g id=\"a_node11\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1032,-68C1032,-68 1002,-68 1002,-68 996,-68 990,-62 990,-56 990,-56 990,-44 990,-44 990,-38 996,-32 1002,-32 1002,-32 1032,-32 1032,-32 1038,-32 1044,-38 1044,-44 1044,-44 1044,-56 1044,-56 1044,-62 1038,-68 1032,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_bn&#45;&gt;layer1_residual_res1_relu -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>layer1_residual_res1_bn&#45;&gt;layer1_residual_res1_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M954.403,-50C962.393,-50 971.311,-50 979.824,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"979.919,-53.5001 989.919,-50 979.919,-46.5001 979.919,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_conv -->\n",
       "<g id=\"node12\" class=\"node\"><title>layer1_residual_res2_conv</title>\n",
       "<g id=\"a_node12\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 128, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1122,-68C1122,-68 1092,-68 1092,-68 1086,-68 1080,-62 1080,-56 1080,-56 1080,-44 1080,-44 1080,-38 1086,-32 1092,-32 1092,-32 1122,-32 1122,-32 1128,-32 1134,-38 1134,-44 1134,-44 1134,-56 1134,-56 1134,-62 1128,-68 1122,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"1107\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_res1_relu&#45;&gt;layer1_residual_res2_conv -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>layer1_residual_res1_relu&#45;&gt;layer1_residual_res2_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.4,-50C1052.39,-50 1061.31,-50 1069.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1069.92,-53.5001 1079.92,-50 1069.92,-46.5001 1069.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_bn -->\n",
       "<g id=\"node13\" class=\"node\"><title>layer1_residual_res2_bn</title>\n",
       "<g id=\"a_node13\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 128, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1212,-68C1212,-68 1182,-68 1182,-68 1176,-68 1170,-62 1170,-56 1170,-56 1170,-44 1170,-44 1170,-38 1176,-32 1182,-32 1182,-32 1212,-32 1212,-32 1218,-32 1224,-38 1224,-44 1224,-44 1224,-56 1224,-56 1224,-62 1218,-68 1212,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_conv&#45;&gt;layer1_residual_res2_bn -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>layer1_residual_res2_conv&#45;&gt;layer1_residual_res2_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1134.4,-50C1142.39,-50 1151.31,-50 1159.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1159.92,-53.5001 1169.92,-50 1159.92,-46.5001 1159.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_relu -->\n",
       "<g id=\"node14\" class=\"node\"><title>layer1_residual_res2_relu</title>\n",
       "<g id=\"a_node14\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1302,-68C1302,-68 1272,-68 1272,-68 1266,-68 1260,-62 1260,-56 1260,-56 1260,-44 1260,-44 1260,-38 1266,-32 1272,-32 1272,-32 1302,-32 1302,-32 1308,-32 1314,-38 1314,-44 1314,-44 1314,-56 1314,-56 1314,-62 1308,-68 1302,-68\"/>\n",
       "<text text-anchor=\"middle\" x=\"1287\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_bn&#45;&gt;layer1_residual_res2_relu -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>layer1_residual_res2_bn&#45;&gt;layer1_residual_res2_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1224.4,-50C1232.39,-50 1241.31,-50 1249.82,-50\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1249.92,-53.5001 1259.92,-50 1249.92,-46.5001 1249.92,-53.5001\"/>\n",
       "</g>\n",
       "<!-- layer1_residual_res2_relu&#45;&gt;layer1_residual_add -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>layer1_residual_res2_relu&#45;&gt;layer1_residual_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1314.4,-56.0007C1322.48,-57.8366 1331.51,-59.8879 1340.1,-61.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1339.39,-65.2692 1349.92,-64.0725 1340.94,-58.4432 1339.39,-65.2692\"/>\n",
       "</g>\n",
       "<!-- layer2_conv -->\n",
       "<g id=\"node16\" class=\"node\"><title>layer2_conv</title>\n",
       "<g id=\"a_node16\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 128, &#39;out_channels&#39;: 256, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1482,-88C1482,-88 1452,-88 1452,-88 1446,-88 1440,-82 1440,-76 1440,-76 1440,-64 1440,-64 1440,-58 1446,-52 1452,-52 1452,-52 1482,-52 1482,-52 1488,-52 1494,-58 1494,-64 1494,-64 1494,-76 1494,-76 1494,-82 1488,-88 1482,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1467\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer1_residual_add&#45;&gt;layer2_conv -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>layer1_residual_add&#45;&gt;layer2_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1404.4,-70C1412.39,-70 1421.31,-70 1429.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1429.92,-73.5001 1439.92,-70 1429.92,-66.5001 1429.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_bn -->\n",
       "<g id=\"node17\" class=\"node\"><title>layer2_bn</title>\n",
       "<g id=\"a_node17\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 256, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1572,-88C1572,-88 1542,-88 1542,-88 1536,-88 1530,-82 1530,-76 1530,-76 1530,-64 1530,-64 1530,-58 1536,-52 1542,-52 1542,-52 1572,-52 1572,-52 1578,-52 1584,-58 1584,-64 1584,-64 1584,-76 1584,-76 1584,-82 1578,-88 1572,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1557\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_conv&#45;&gt;layer2_bn -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>layer2_conv&#45;&gt;layer2_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1494.4,-70C1502.39,-70 1511.31,-70 1519.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1519.92,-73.5001 1529.92,-70 1519.92,-66.5001 1519.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_relu -->\n",
       "<g id=\"node18\" class=\"node\"><title>layer2_relu</title>\n",
       "<g id=\"a_node18\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M1662,-88C1662,-88 1632,-88 1632,-88 1626,-88 1620,-82 1620,-76 1620,-76 1620,-64 1620,-64 1620,-58 1626,-52 1632,-52 1632,-52 1662,-52 1662,-52 1668,-52 1674,-58 1674,-64 1674,-64 1674,-76 1674,-76 1674,-82 1668,-88 1662,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1647\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_bn&#45;&gt;layer2_relu -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>layer2_bn&#45;&gt;layer2_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1584.4,-70C1592.39,-70 1601.31,-70 1609.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1609.92,-73.5001 1619.92,-70 1609.92,-66.5001 1609.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer2_pool -->\n",
       "<g id=\"node19\" class=\"node\"><title>layer2_pool</title>\n",
       "<g id=\"a_node19\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M1752,-88C1752,-88 1722,-88 1722,-88 1716,-88 1710,-82 1710,-76 1710,-76 1710,-64 1710,-64 1710,-58 1716,-52 1722,-52 1722,-52 1752,-52 1752,-52 1758,-52 1764,-58 1764,-64 1764,-64 1764,-76 1764,-76 1764,-82 1758,-88 1752,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1737\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_relu&#45;&gt;layer2_pool -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>layer2_relu&#45;&gt;layer2_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1674.4,-70C1682.39,-70 1691.31,-70 1699.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1699.92,-73.5001 1709.92,-70 1699.92,-66.5001 1699.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_conv -->\n",
       "<g id=\"node20\" class=\"node\"><title>layer3_conv</title>\n",
       "<g id=\"a_node20\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 256, &#39;out_channels&#39;: 512, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M1842,-88C1842,-88 1812,-88 1812,-88 1806,-88 1800,-82 1800,-76 1800,-76 1800,-64 1800,-64 1800,-58 1806,-52 1812,-52 1812,-52 1842,-52 1842,-52 1848,-52 1854,-58 1854,-64 1854,-64 1854,-76 1854,-76 1854,-82 1848,-88 1842,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1827\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer2_pool&#45;&gt;layer3_conv -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>layer2_pool&#45;&gt;layer3_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1764.4,-70C1772.39,-70 1781.31,-70 1789.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1789.92,-73.5001 1799.92,-70 1789.92,-66.5001 1789.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_bn -->\n",
       "<g id=\"node21\" class=\"node\"><title>layer3_bn</title>\n",
       "<g id=\"a_node21\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 512, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M1932,-88C1932,-88 1902,-88 1902,-88 1896,-88 1890,-82 1890,-76 1890,-76 1890,-64 1890,-64 1890,-58 1896,-52 1902,-52 1902,-52 1932,-52 1932,-52 1938,-52 1944,-58 1944,-64 1944,-64 1944,-76 1944,-76 1944,-82 1938,-88 1932,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"1917\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_conv&#45;&gt;layer3_bn -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>layer3_conv&#45;&gt;layer3_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1854.4,-70C1862.39,-70 1871.31,-70 1879.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1879.92,-73.5001 1889.92,-70 1879.92,-66.5001 1879.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_relu -->\n",
       "<g id=\"node22\" class=\"node\"><title>layer3_relu</title>\n",
       "<g id=\"a_node22\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2022,-88C2022,-88 1992,-88 1992,-88 1986,-88 1980,-82 1980,-76 1980,-76 1980,-64 1980,-64 1980,-58 1986,-52 1992,-52 1992,-52 2022,-52 2022,-52 2028,-52 2034,-58 2034,-64 2034,-64 2034,-76 2034,-76 2034,-82 2028,-88 2022,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"2007\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_bn&#45;&gt;layer3_relu -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>layer3_bn&#45;&gt;layer3_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1944.4,-70C1952.39,-70 1961.31,-70 1969.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1969.92,-73.5001 1979.92,-70 1969.92,-66.5001 1969.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_pool -->\n",
       "<g id=\"node23\" class=\"node\"><title>layer3_pool</title>\n",
       "<g id=\"a_node23\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 2, &#39;stride&#39;: 2, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2112,-88C2112,-88 2082,-88 2082,-88 2076,-88 2070,-82 2070,-76 2070,-76 2070,-64 2070,-64 2070,-58 2076,-52 2082,-52 2082,-52 2112,-52 2112,-52 2118,-52 2124,-58 2124,-64 2124,-64 2124,-76 2124,-76 2124,-82 2118,-88 2112,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"2097\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_relu&#45;&gt;layer3_pool -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>layer3_relu&#45;&gt;layer3_pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2034.4,-70C2042.39,-70 2051.31,-70 2059.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2059.92,-73.5001 2069.92,-70 2059.92,-66.5001 2059.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_in -->\n",
       "<g id=\"node24\" class=\"node\"><title>layer3_residual_in</title>\n",
       "<g id=\"a_node24\"><a xlink:title=\"Identity {}\">\n",
       "<path fill=\"#ccebc5\" stroke=\"black\" d=\"M2202,-88C2202,-88 2172,-88 2172,-88 2166,-88 2160,-82 2160,-76 2160,-76 2160,-64 2160,-64 2160,-58 2166,-52 2172,-52 2172,-52 2202,-52 2202,-52 2208,-52 2214,-58 2214,-64 2214,-64 2214,-76 2214,-76 2214,-82 2208,-88 2202,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"2187\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">in</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_pool&#45;&gt;layer3_residual_in -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>layer3_pool&#45;&gt;layer3_residual_in</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2124.4,-70C2132.39,-70 2141.31,-70 2149.82,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2149.92,-73.5001 2159.92,-70 2149.92,-66.5001 2149.92,-73.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_conv -->\n",
       "<g id=\"node25\" class=\"node\"><title>layer3_residual_res1_conv</title>\n",
       "<g id=\"a_node25\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 512, &#39;out_channels&#39;: 512, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2292,-78C2292,-78 2262,-78 2262,-78 2256,-78 2250,-72 2250,-66 2250,-66 2250,-54 2250,-54 2250,-48 2256,-42 2262,-42 2262,-42 2292,-42 2292,-42 2298,-42 2304,-48 2304,-54 2304,-54 2304,-66 2304,-66 2304,-72 2298,-78 2292,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2277\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_in&#45;&gt;layer3_residual_res1_conv -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>layer3_residual_in&#45;&gt;layer3_residual_res1_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2214.4,-66.9997C2222.39,-66.0917 2231.31,-65.0783 2239.82,-64.1109\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2240.38,-67.5706 2249.92,-62.9638 2239.59,-60.6153 2240.38,-67.5706\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_add -->\n",
       "<g id=\"node31\" class=\"node\"><title>layer3_residual_add</title>\n",
       "<g id=\"a_node31\"><a xlink:title=\"Add {}\">\n",
       "<path fill=\"#ffed6f\" stroke=\"black\" d=\"M2832,-98C2832,-98 2802,-98 2802,-98 2796,-98 2790,-92 2790,-86 2790,-86 2790,-74 2790,-74 2790,-68 2796,-62 2802,-62 2802,-62 2832,-62 2832,-62 2838,-62 2844,-68 2844,-74 2844,-74 2844,-86 2844,-86 2844,-92 2838,-98 2832,-98\"/>\n",
       "<text text-anchor=\"middle\" x=\"2817\" y=\"-76.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_in&#45;&gt;layer3_residual_add -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>layer3_residual_in&#45;&gt;layer3_residual_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2205.16,-88.3721C2214.97,-97.5647 2228.07,-107.815 2242,-113 2293.69,-132.244 2310.84,-118 2366,-118 2366,-118 2366,-118 2638,-118 2693.16,-118 2709.22,-129.006 2762,-113 2769.3,-110.786 2776.61,-107.345 2783.4,-103.506\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2785.55,-106.295 2792.26,-98.1018 2781.9,-100.318 2785.55,-106.295\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_bn -->\n",
       "<g id=\"node26\" class=\"node\"><title>layer3_residual_res1_bn</title>\n",
       "<g id=\"a_node26\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 512, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2382,-78C2382,-78 2352,-78 2352,-78 2346,-78 2340,-72 2340,-66 2340,-66 2340,-54 2340,-54 2340,-48 2346,-42 2352,-42 2352,-42 2382,-42 2382,-42 2388,-42 2394,-48 2394,-54 2394,-54 2394,-66 2394,-66 2394,-72 2388,-78 2382,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2367\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_conv&#45;&gt;layer3_residual_res1_bn -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>layer3_residual_res1_conv&#45;&gt;layer3_residual_res1_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2304.4,-60C2312.39,-60 2321.31,-60 2329.82,-60\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2329.92,-63.5001 2339.92,-60 2329.92,-56.5001 2329.92,-63.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_relu -->\n",
       "<g id=\"node27\" class=\"node\"><title>layer3_residual_res1_relu</title>\n",
       "<g id=\"a_node27\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2472,-78C2472,-78 2442,-78 2442,-78 2436,-78 2430,-72 2430,-66 2430,-66 2430,-54 2430,-54 2430,-48 2436,-42 2442,-42 2442,-42 2472,-42 2472,-42 2478,-42 2484,-48 2484,-54 2484,-54 2484,-66 2484,-66 2484,-72 2478,-78 2472,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2457\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_bn&#45;&gt;layer3_residual_res1_relu -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>layer3_residual_res1_bn&#45;&gt;layer3_residual_res1_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2394.4,-60C2402.39,-60 2411.31,-60 2419.82,-60\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2419.92,-63.5001 2429.92,-60 2419.92,-56.5001 2419.92,-63.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_conv -->\n",
       "<g id=\"node28\" class=\"node\"><title>layer3_residual_res2_conv</title>\n",
       "<g id=\"a_node28\"><a xlink:title=\"Conv2d {&#39;in_channels&#39;: 512, &#39;out_channels&#39;: 512, &#39;kernel_size&#39;: (3, 3), &#39;stride&#39;: (1, 1), &#39;padding&#39;: (1, 1), &#39;dilation&#39;: (1, 1), &#39;groups&#39;: 1, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#bebada\" stroke=\"black\" d=\"M2562,-78C2562,-78 2532,-78 2532,-78 2526,-78 2520,-72 2520,-66 2520,-66 2520,-54 2520,-54 2520,-48 2526,-42 2532,-42 2532,-42 2562,-42 2562,-42 2568,-42 2574,-48 2574,-54 2574,-54 2574,-66 2574,-66 2574,-72 2568,-78 2562,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2547\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">conv</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_res1_relu&#45;&gt;layer3_residual_res2_conv -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>layer3_residual_res1_relu&#45;&gt;layer3_residual_res2_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2484.4,-60C2492.39,-60 2501.31,-60 2509.82,-60\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2509.92,-63.5001 2519.92,-60 2509.92,-56.5001 2509.92,-63.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_bn -->\n",
       "<g id=\"node29\" class=\"node\"><title>layer3_residual_res2_bn</title>\n",
       "<g id=\"a_node29\"><a xlink:title=\"BatchNorm2d {&#39;num_features&#39;: 512, &#39;eps&#39;: 1e&#45;05, &#39;momentum&#39;: 0.1, &#39;affine&#39;: True, &#39;track_running_stats&#39;: True}\">\n",
       "<path fill=\"#ffffb3\" stroke=\"black\" d=\"M2652,-78C2652,-78 2622,-78 2622,-78 2616,-78 2610,-72 2610,-66 2610,-66 2610,-54 2610,-54 2610,-48 2616,-42 2622,-42 2622,-42 2652,-42 2652,-42 2658,-42 2664,-48 2664,-54 2664,-54 2664,-66 2664,-66 2664,-72 2658,-78 2652,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2637\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">bn</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_conv&#45;&gt;layer3_residual_res2_bn -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>layer3_residual_res2_conv&#45;&gt;layer3_residual_res2_bn</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2574.4,-60C2582.39,-60 2591.31,-60 2599.82,-60\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2599.92,-63.5001 2609.92,-60 2599.92,-56.5001 2599.92,-63.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_relu -->\n",
       "<g id=\"node30\" class=\"node\"><title>layer3_residual_res2_relu</title>\n",
       "<g id=\"a_node30\"><a xlink:title=\"ReLU {&#39;inplace&#39;: True}\">\n",
       "<path fill=\"#fb8072\" stroke=\"black\" d=\"M2742,-78C2742,-78 2712,-78 2712,-78 2706,-78 2700,-72 2700,-66 2700,-66 2700,-54 2700,-54 2700,-48 2706,-42 2712,-42 2712,-42 2742,-42 2742,-42 2748,-42 2754,-48 2754,-54 2754,-54 2754,-66 2754,-66 2754,-72 2748,-78 2742,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"2727\" y=\"-56.3\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_bn&#45;&gt;layer3_residual_res2_relu -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>layer3_residual_res2_bn&#45;&gt;layer3_residual_res2_relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2664.4,-60C2672.39,-60 2681.31,-60 2689.82,-60\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2689.92,-63.5001 2699.92,-60 2689.92,-56.5001 2689.92,-63.5001\"/>\n",
       "</g>\n",
       "<!-- layer3_residual_res2_relu&#45;&gt;layer3_residual_add -->\n",
       "<g id=\"edge33\" class=\"edge\"><title>layer3_residual_res2_relu&#45;&gt;layer3_residual_add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2754.4,-66.0007C2762.48,-67.8366 2771.51,-69.8879 2780.1,-71.842\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2779.39,-75.2692 2789.92,-74.0725 2780.94,-68.4432 2779.39,-75.2692\"/>\n",
       "</g>\n",
       "<!-- pool -->\n",
       "<g id=\"node32\" class=\"node\"><title>pool</title>\n",
       "<g id=\"a_node32\"><a xlink:title=\"MaxPool2d {&#39;kernel_size&#39;: 4, &#39;stride&#39;: 4, &#39;padding&#39;: 0, &#39;dilation&#39;: 1, &#39;return_indices&#39;: False, &#39;ceil_mode&#39;: False}\">\n",
       "<path fill=\"#8dd3c7\" stroke=\"black\" d=\"M2922,-98C2922,-98 2892,-98 2892,-98 2886,-98 2880,-92 2880,-86 2880,-86 2880,-74 2880,-74 2880,-68 2886,-62 2892,-62 2892,-62 2922,-62 2922,-62 2928,-62 2934,-68 2934,-74 2934,-74 2934,-86 2934,-86 2934,-92 2928,-98 2922,-98\"/>\n",
       "<text text-anchor=\"middle\" x=\"2907\" y=\"-76.3\" font-family=\"Times,serif\" font-size=\"14.00\">pool</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- layer3_residual_add&#45;&gt;pool -->\n",
       "<g id=\"edge34\" class=\"edge\"><title>layer3_residual_add&#45;&gt;pool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2844.4,-80C2852.39,-80 2861.31,-80 2869.82,-80\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2869.92,-83.5001 2879.92,-80 2869.92,-76.5001 2869.92,-83.5001\"/>\n",
       "</g>\n",
       "<!-- flatten -->\n",
       "<g id=\"node33\" class=\"node\"><title>flatten</title>\n",
       "<g id=\"a_node33\"><a xlink:title=\"Flatten {}\">\n",
       "<path fill=\"#b3de69\" stroke=\"black\" d=\"M3012,-98C3012,-98 2982,-98 2982,-98 2976,-98 2970,-92 2970,-86 2970,-86 2970,-74 2970,-74 2970,-68 2976,-62 2982,-62 2982,-62 3012,-62 3012,-62 3018,-62 3024,-68 3024,-74 3024,-74 3024,-86 3024,-86 3024,-92 3018,-98 3012,-98\"/>\n",
       "<text text-anchor=\"middle\" x=\"2997\" y=\"-76.3\" font-family=\"Times,serif\" font-size=\"14.00\">flatten</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pool&#45;&gt;flatten -->\n",
       "<g id=\"edge35\" class=\"edge\"><title>pool&#45;&gt;flatten</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2934.4,-80C2942.39,-80 2951.31,-80 2959.82,-80\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2959.92,-83.5001 2969.92,-80 2959.92,-76.5001 2959.92,-83.5001\"/>\n",
       "</g>\n",
       "<!-- linear -->\n",
       "<g id=\"node34\" class=\"node\"><title>linear</title>\n",
       "<g id=\"a_node34\"><a xlink:title=\"Linear {&#39;in_features&#39;: 512, &#39;out_features&#39;: 10, &#39;bias&#39;: None}\">\n",
       "<path fill=\"#fccde5\" stroke=\"black\" d=\"M3102,-98C3102,-98 3072,-98 3072,-98 3066,-98 3060,-92 3060,-86 3060,-86 3060,-74 3060,-74 3060,-68 3066,-62 3072,-62 3072,-62 3102,-62 3102,-62 3108,-62 3114,-68 3114,-74 3114,-74 3114,-86 3114,-86 3114,-92 3108,-98 3102,-98\"/>\n",
       "<text text-anchor=\"middle\" x=\"3087\" y=\"-76.3\" font-family=\"Times,serif\" font-size=\"14.00\">linear</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- flatten&#45;&gt;linear -->\n",
       "<g id=\"edge36\" class=\"edge\"><title>flatten&#45;&gt;linear</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3024.4,-80C3032.39,-80 3041.31,-80 3049.82,-80\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3049.92,-83.5001 3059.92,-80 3049.92,-76.5001 3049.92,-83.5001\"/>\n",
       "</g>\n",
       "<!-- classifier -->\n",
       "<g id=\"node35\" class=\"node\"><title>classifier</title>\n",
       "<g id=\"a_node35\"><a xlink:title=\"Mul {&#39;weight&#39;: 0.125}\">\n",
       "<path fill=\"#bc80bd\" stroke=\"black\" d=\"M3203,-98C3203,-98 3162,-98 3162,-98 3156,-98 3150,-92 3150,-86 3150,-86 3150,-74 3150,-74 3150,-68 3156,-62 3162,-62 3162,-62 3203,-62 3203,-62 3209,-62 3215,-68 3215,-74 3215,-74 3215,-86 3215,-86 3215,-92 3209,-98 3203,-98\"/>\n",
       "<text text-anchor=\"middle\" x=\"3182.5\" y=\"-76.3\" font-family=\"Times,serif\" font-size=\"14.00\">classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- linear&#45;&gt;classifier -->\n",
       "<g id=\"edge37\" class=\"edge\"><title>linear&#45;&gt;classifier</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M3114.04,-80C3122.05,-80 3131.06,-80 3139.83,-80\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"3139.95,-83.5001 3149.94,-80 3139.94,-76.5001 3139.95,-83.5001\"/>\n",
       "</g>\n",
       "<!-- input -->\n",
       "<g id=\"node36\" class=\"node\"><title>input</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M42,-88C42,-88 12,-88 12,-88 6,-88 0,-82 0,-76 0,-76 0,-64 0,-64 0,-58 6,-52 12,-52 12,-52 42,-52 42,-52 48,-52 54,-58 54,-64 54,-64 54,-76 54,-76 54,-82 48,-88 42,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-66.3\" font-family=\"Times,serif\" font-size=\"14.00\">input</text>\n",
       "</g>\n",
       "<!-- input&#45;&gt;prep_conv -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>input&#45;&gt;prep_conv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M54.4029,-70C62.3932,-70 71.3106,-70 79.8241,-70\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.919,-73.5001 89.919,-70 79.919,-66.5001 79.919,-73.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<core.DotGraph at 0x7fcd233a26d8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       epoch           lr   train time   train loss    train acc    test time    test loss     test acc   total time\n",
      "           1       0.0800       2.9545       1.6042       0.4199       0.2184       1.1629       0.5758       3.1729\n",
      "           2       0.1600       2.9604       0.9170       0.6724       0.2195       0.9615       0.6635       6.3529\n",
      "           3       0.2400       3.0128       0.7207       0.7483       0.2183       0.7565       0.7382       9.5839\n",
      "           4       0.3200       2.9565       0.6183       0.7838       0.2188       1.1623       0.6477      12.7593\n",
      "           5       0.4000       2.9525       0.5530       0.8088       0.2182       1.0094       0.6839      15.9300\n",
      "           6       0.3789       2.9570       0.4969       0.8286       0.2180       0.5447       0.8138      19.1050\n",
      "           7       0.3579       2.9556       0.4450       0.8492       0.2656       0.7271       0.7555      22.3261\n",
      "           8       0.3368       2.9508       0.4059       0.8614       0.2176       0.6534       0.7797      25.4946\n",
      "           9       0.3158       2.9494       0.3859       0.8686       0.2179       0.4609       0.8481      28.6619\n",
      "          10       0.2947       2.9494       0.3639       0.8767       0.2179       0.6188       0.7887      31.8292\n",
      "          11       0.2737       2.9531       0.3453       0.8818       0.2662       0.4030       0.8628      35.0485\n",
      "          12       0.2526       2.9552       0.3228       0.8906       0.2178       0.7032       0.7801      38.2215\n",
      "          13       0.2316       2.9556       0.3061       0.8957       0.2174       0.3760       0.8690      41.3945\n",
      "          14       0.2105       2.9527       0.2847       0.9040       0.2180       0.3742       0.8726      44.5652\n",
      "          15       0.1895       2.9544       0.2721       0.9086       0.2721       0.3820       0.8720      47.7917\n",
      "          16       0.1684       2.9506       0.2458       0.9176       0.2177       0.3834       0.8668      50.9599\n",
      "          17       0.1474       2.9526       0.2333       0.9214       0.2177       0.3838       0.8728      54.1303\n",
      "          18       0.1263       2.9499       0.2115       0.9291       0.2181       0.4067       0.8635      57.2982\n",
      "          19       0.1053       2.9474       0.1860       0.9379       0.2673       0.3779       0.8805      60.5129\n",
      "          20       0.0842       2.9511       0.1646       0.9457       0.2178       0.3980       0.8770      63.6817\n",
      "          21       0.0632       2.9505       0.1385       0.9558       0.2180       0.2588       0.9171      66.8502\n",
      "          22       0.0421       2.9529       0.1163       0.9629       0.2182       0.2086       0.9312      70.0213\n",
      "          23       0.0211       2.9526       0.0917       0.9727       0.2175       0.1857       0.9378      73.1914\n",
      "          24       0.0000       3.0010       0.0732       0.9797       0.2175       0.1744       0.9413      76.4099\n"
     ]
    }
   ],
   "source": [
    "lr_schedule = PiecewiseLinear([0, 5, 24], [0, 0.4, 0])\n",
    "batch_size = 512\n",
    "\n",
    "n = net()\n",
    "draw(build_graph(n))\n",
    "model = Network(n).to(device).half()\n",
    "train_set_x = Transform(train_set, [Crop(32, 32), FlipLR(), Cutout(8,8)])\n",
    "summary = train(model, lr_schedule, train_set_x, test_set, batch_size=batch_size, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
